Fix log parsing: JSON arrays, Zeek format, and beacon detection priority

c2sentinel.py

@@ -1051,14 +1051,18 @@ class LogParser:
     def parse_zeek_conn(log_line: str) -> Optional[Dict]:
         """Parse Zeek/Bro conn.log format."""
         try:
+            # Skip header lines
+            if log_line.startswith('#'):
+                return None
             parts = log_line.strip().split('\t')
-            if len(parts) >= 15:
+            # Minimum fields: ts, uid, orig_h, orig_p, resp_h, resp_p, proto, service, duration, orig_bytes, resp_bytes
+            if len(parts) >= 11:
                 return {
                     'timestamp': float(parts[0]),
                     'src_ip': parts[2],
-                    'src_port': int(parts[3]),
+                    'src_port': int(parts[3]) if parts[3] != '-' else 0,
                     'dst_ip': parts[4],
-                    'dst_port': int(parts[5]),
+                    'dst_port': int(parts[5]) if parts[5] != '-' else 0,
                     'protocol': parts[6],
                     'duration': float(parts[8]) if parts[8] != '-' else 0,
                     'bytes_sent': int(parts[9]) if parts[9] != '-' else 0,
@@ -1518,6 +1522,7 @@ class C2Sentinel:
         # PHASE 4: Behavioral refinement
         # ================================================================
 
+        beacon_indicators = 0  # Initialize here so it's always defined
         dst_ips = set(conn.get('dst_ip', '') for conn in connections)
         bytes_recv = [conn.get('bytes_recv', 0) for conn in connections]
         bytes_sent = [conn.get('bytes_sent', 0) for conn in connections]
@@ -1567,15 +1572,63 @@ class C2Sentinel:
                     c2_prob = min(1.0, c2_prob * 1.5)
                     result.risk_factors.append("APT-style slow beacon pattern")
 
+                # ============================================================
+                # CRITICAL: Explicit beacon pattern override
+                # When ALL classic beacon indicators are present, force detection
+                # ============================================================
+                beacon_indicators = 0
+
+                # Indicator 1: Very regular timing (CV < 0.15)
+                if interval_cv < 0.15:
+                    beacon_indicators += 1
+
+                # Indicator 2: Very consistent sizes (both sent and recv CV < 0.15)
+                if recv_cv < 0.15 and sent_cv < 0.15:
+                    beacon_indicators += 1
+
+                # Indicator 3: Small packet sizes (typical heartbeat)
+                mean_sent = np.mean(bytes_sent) if bytes_sent else 0
+                mean_recv = np.mean(bytes_recv) if bytes_recv else 0
+                if mean_sent < 500 and mean_recv < 500:
+                    beacon_indicators += 1
+
+                # Indicator 4: Regular interval in beacon range (5s - 300s)
+                if 5 <= mean_interval <= 300:
+                    beacon_indicators += 1
+
+                # Indicator 5: Sufficient sample size
+                if len(connections) >= 8:
+                    beacon_indicators += 1
+
+                # If 4+ of 5 indicators present, this is almost certainly C2
+                if beacon_indicators >= 4:
+                    c2_prob = max(c2_prob, 0.85)  # Force high probability
+                    result.risk_factors.append(f"Classic C2 beacon pattern detected ({beacon_indicators}/5 indicators)")
+                    result.detection_method = DetectionMethod.BEHAVIORAL.value
+                elif beacon_indicators >= 3:
+                    c2_prob = max(c2_prob, 0.65)  # Likely C2
+                    result.risk_factors.append(f"Probable C2 beacon pattern ({beacon_indicators}/5 indicators)")
+
         # ================================================================
-        # PHASE 5: Apply legitimate pattern discount
+        # PHASE 5: Apply legitimate pattern discount (but respect strong beacon signals)
         # ================================================================
 
         if matched_pattern and pattern_confidence > 0.5:
-            # Reduce probability based on pattern match
-            discount = 1.0 - (pattern_confidence * 0.7)  # Up to 70% reduction
-            c2_prob *= discount
-            result.mitigating_factors.append(f"Legitimate pattern match reduces probability by {(1-discount)*100:.0f}%")
+            # If strong beacon indicators present, skip or reduce the legitimate pattern discount
+            # This prevents false negatives when C2 mimics legitimate patterns
+            if beacon_indicators >= 4:
+                # Strong C2 signal - don't discount
+                result.mitigating_factors.append(f"Matches {matched_pattern.name} pattern but beacon indicators override")
+            elif beacon_indicators >= 3:
+                # Moderate C2 signal - apply reduced discount
+                discount = 1.0 - (pattern_confidence * 0.3)  # Max 30% reduction
+                c2_prob *= discount
+                result.mitigating_factors.append(f"Legitimate pattern match reduces probability by {(1-discount)*100:.0f}%")
+            else:
+                # Weak C2 signal - apply full discount
+                discount = 1.0 - (pattern_confidence * 0.7)  # Up to 70% reduction
+                c2_prob *= discount
+                result.mitigating_factors.append(f"Legitimate pattern match reduces probability by {(1-discount)*100:.0f}%")
 
         # ================================================================
         # PHASE 6: Apply context inference (always check whitelist/blacklist)
@@ -1667,14 +1720,40 @@ class C2Sentinel:
     ) -> List[Dict]:
         """Analyze raw log lines for C2 activity."""
         connections = []
-        for line in log_lines:
-            conn = self.log_parser.parse_json(line)
-            if not conn:
-                conn = self.log_parser.parse_zeek_conn(line)
-            if not conn:
-                conn = self.log_parser.parse_syslog(line)
-            if conn:
-                connections.append(conn)
+
+        # First try to parse as a complete JSON array
+        full_content = ''.join(log_lines)
+        try:
+            data = json.loads(full_content)
+            if isinstance(data, list):
+                for item in data:
+                    if isinstance(item, dict):
+                        conn = {
+                            'timestamp': item.get('timestamp', item.get('@timestamp', 0)),
+                            'src_ip': item.get('src_ip', item.get('source_ip', '')),
+                            'dst_ip': item.get('dst_ip', item.get('dest_ip', '')),
+                            'src_port': int(item.get('src_port', item.get('source_port', 0))),
+                            'dst_port': int(item.get('dst_port', item.get('dest_port', 0))),
+                            'protocol': item.get('protocol', 'tcp'),
+                            'bytes_sent': int(item.get('bytes_sent', item.get('bytes_out', 0))),
+                            'bytes_recv': int(item.get('bytes_recv', item.get('bytes_in', 0))),
+                            'duration': float(item.get('duration', 0))
+                        }
+                        if conn.get('dst_ip'):
+                            connections.append(conn)
+        except (json.JSONDecodeError, TypeError):
+            pass
+
+        # Fall back to line-by-line parsing
+        if not connections:
+            for line in log_lines:
+                conn = self.log_parser.parse_json(line)
+                if not conn:
+                    conn = self.log_parser.parse_zeek_conn(line)
+                if not conn:
+                    conn = self.log_parser.parse_syslog(line)
+                if conn:
+                    connections.append(conn)
 
         if not connections:
             return []