Upload folder using huggingface_hub

app.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""C2Sentinel - HuggingFace Space Demo"""
+"""C2Sentinel Demo"""
 
 import gradio as gr
 import json
@@ -7,321 +7,153 @@ import re
 from datetime import datetime
 from huggingface_hub import hf_hub_download
 
-# Download and load model
+# Load model
 hf_hub_download(repo_id="danielostrow/c2sentinel", filename="c2sentinel.py", local_dir=".")
 hf_hub_download(repo_id="danielostrow/c2sentinel", filename="c2_sentinel.safetensors", local_dir=".")
 hf_hub_download(repo_id="danielostrow/c2sentinel", filename="c2_sentinel.json", local_dir=".")
 from c2sentinel import C2Sentinel
 sentinel = C2Sentinel.load('c2_sentinel')
 
-# Examples
 EXAMPLES = {
-    "C2 Beacon (60s)": [
-        {"timestamp": 1705600000+i*60, "dst_ip": "45.33.32.156", "dst_port": 443, "bytes_sent": 200, "bytes_recv": 500}
-        for i in range(8)
-    ],
-    "Metasploit 4444": [
-        {"timestamp": 1705600000+i*30, "dst_ip": "10.10.10.10", "dst_port": 4444, "bytes_sent": 150, "bytes_recv": 300}
-        for i in range(6)
-    ],
-    "SSH Keepalive": [
-        {"timestamp": 1705600000+i*30, "dst_ip": "192.168.1.10", "dst_port": 22, "bytes_sent": 48, "bytes_recv": 48}
-        for i in range(6)
-    ],
-    "Web Traffic": [
-        {"timestamp": 1705600000, "dst_ip": "93.184.216.34", "dst_port": 443, "bytes_sent": 500, "bytes_recv": 15000},
-        {"timestamp": 1705600002, "dst_ip": "151.101.1.140", "dst_port": 443, "bytes_sent": 800, "bytes_recv": 45000},
-        {"timestamp": 1705600010, "dst_ip": "172.217.14.206", "dst_port": 443, "bytes_sent": 300, "bytes_recv": 12000},
-        {"timestamp": 1705600015, "dst_ip": "93.184.216.34", "dst_port": 443, "bytes_sent": 200, "bytes_recv": 8000},
-        {"timestamp": 1705600025, "dst_ip": "151.101.1.140", "dst_port": 443, "bytes_sent": 150, "bytes_recv": 5000},
-    ],
+    "C2 Beacon": [{"timestamp": 1705600000+i*60, "dst_ip": "45.33.32.156", "dst_port": 443, "bytes_sent": 200, "bytes_recv": 500} for i in range(8)],
+    "Metasploit": [{"timestamp": 1705600000+i*30, "dst_ip": "10.10.10.10", "dst_port": 4444, "bytes_sent": 150, "bytes_recv": 300} for i in range(6)],
+    "SSH Keepalive": [{"timestamp": 1705600000+i*30, "dst_ip": "192.168.1.10", "dst_port": 22, "bytes_sent": 48, "bytes_recv": 48} for i in range(6)],
+    "Web Traffic": [{"timestamp": 1705600000+i*5, "dst_ip": f"93.184.{i}.34", "dst_port": 443, "bytes_sent": 500+i*100, "bytes_recv": 15000+i*5000} for i in range(5)],
 }
 
 
-def parse_timestamp(ts_str):
-    """Parse various timestamp formats to unix timestamp."""
-    if not ts_str:
-        return None
-
-    # Already numeric
+def parse_ts(s):
+    if not s: return None
     try:
-        ts = float(ts_str)
-        if ts > 1e12:  # milliseconds
-            return ts / 1000
-        return ts
-    except:
-        pass
-
-    # Common date formats
-    formats = [
-        "%Y-%m-%dT%H:%M:%S.%fZ",
-        "%Y-%m-%dT%H:%M:%SZ",
-        "%Y-%m-%dT%H:%M:%S.%f",
-        "%Y-%m-%dT%H:%M:%S",
-        "%Y-%m-%d %H:%M:%S.%f",
-        "%Y-%m-%d %H:%M:%S",
-        "%b %d %H:%M:%S",
-        "%b  %d %H:%M:%S",
-        "%d/%b/%Y:%H:%M:%S",
-    ]
-
-    for fmt in formats:
+        t = float(s)
+        return t/1000 if t > 1e12 else t
+    except: pass
+    for f in ["%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%b %d %H:%M:%S"]:
         try:
-            dt = datetime.strptime(ts_str.strip(), fmt)
-            if dt.year == 1900:
-                dt = dt.replace(year=datetime.now().year)
+            dt = datetime.strptime(str(s).strip(), f)
+            if dt.year == 1900: dt = dt.replace(year=2026)
             return dt.timestamp()
-        except:
-            continue
+        except: pass
     return None
 
 
-def extract_ip_port(text):
-    """Extract IP and port from various formats."""
-    # IP:port format
-    m = re.search(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)', text)
-    if m:
-        return m.group(1), int(m.group(2))
-
-    # Just IP
-    m = re.search(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', text)
-    if m:
-        return m.group(1), None
-
-    return None, None
-
-
 def parse_logs(content):
-    """Parse multiple log formats into connection records."""
-    connections = []
-    lines = content.strip().split('\n')
-
-    for line in lines:
+    conns = []
+    for line in content.strip().split('\n'):
         line = line.strip()
-        if not line or line.startswith('#'):
-            continue
-
-        conn = None
+        if not line or line.startswith('#'): continue
+        c = None
 
-        # 1. JSON format
+        # JSON
         try:
-            obj = json.loads(line)
-            conn = {
-                'timestamp': parse_timestamp(obj.get('timestamp') or obj.get('ts') or obj.get('@timestamp') or obj.get('EventTime') or obj.get('time')),
-                'dst_ip': obj.get('dst_ip') or obj.get('dest_ip') or obj.get('id.resp_h') or obj.get('DestinationIP') or obj.get('dst') or obj.get('destination_ip'),
-                'dst_port': obj.get('dst_port') or obj.get('dest_port') or obj.get('id.resp_p') or obj.get('DestinationPort') or obj.get('dport'),
-                'bytes_sent': obj.get('bytes_sent') or obj.get('orig_bytes') or obj.get('SentBytes') or obj.get('out_bytes') or 0,
-                'bytes_recv': obj.get('bytes_recv') or obj.get('resp_bytes') or obj.get('ReceivedBytes') or obj.get('in_bytes') or 0,
+            o = json.loads(line)
+            c = {
+                'timestamp': parse_ts(o.get('timestamp') or o.get('ts') or o.get('@timestamp') or o.get('time')),
+                'dst_ip': o.get('dst_ip') or o.get('dest_ip') or o.get('id.resp_h') or o.get('DestinationIP') or o.get('dst'),
+                'dst_port': o.get('dst_port') or o.get('dest_port') or o.get('id.resp_p') or o.get('DestinationPort') or o.get('dport'),
+                'bytes_sent': o.get('bytes_sent') or o.get('orig_bytes') or o.get('SentBytes') or 100,
+                'bytes_recv': o.get('bytes_recv') or o.get('resp_bytes') or o.get('ReceivedBytes') or 100,
             }
-        except json.JSONDecodeError:
-            pass
+        except: pass
 
-        # 2. Zeek/Bro conn.log (tab-separated)
-        if not conn:
-            parts = line.split('\t')
-            if len(parts) >= 10:
+        # Zeek
+        if not c:
+            p = line.split('\t')
+            if len(p) >= 10:
                 try:
-                    conn = {
-                        'timestamp': parse_timestamp(parts[0]),
-                        'dst_ip': parts[4] if parts[4] != '-' else None,
-                        'dst_port': int(parts[5]) if parts[5] != '-' else None,
-                        'bytes_sent': int(parts[9]) if len(parts) > 9 and parts[9] != '-' else 0,
-                        'bytes_recv': int(parts[10]) if len(parts) > 10 and parts[10] != '-' else 0,
-                    }
-                except:
-                    pass
-
-        # 3. Syslog format (various)
-        if not conn:
-            # Standard syslog: "Mon DD HH:MM:SS host process: message"
-            syslog_match = re.match(r'^(\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+\S+\s+\S+:\s*(.*)$', line)
-            if syslog_match:
-                ts = parse_timestamp(syslog_match.group(1))
-                msg = syslog_match.group(2)
-                dst_ip, dst_port = extract_ip_port(msg)
-
-                bytes_match = re.search(r'(\d+)\s*bytes?', msg, re.I)
-                bytes_val = int(bytes_match.group(1)) if bytes_match else 100
-
-                if dst_ip:
-                    conn = {
-                        'timestamp': ts,
-                        'dst_ip': dst_ip,
-                        'dst_port': dst_port or 443,
-                        'bytes_sent': bytes_val // 2,
-                        'bytes_recv': bytes_val // 2,
-                    }
-
-        # 4. Windows Event Log (XML-ish or key=value)
-        if not conn:
-            # Key=value format
-            if '=' in line:
-                kv = dict(re.findall(r'(\w+)=("[^"]*"|\S+)', line))
-                for k in list(kv.keys()):
-                    kv[k] = kv[k].strip('"')
-
-                dst_ip = kv.get('DestAddress') or kv.get('DestinationIp') or kv.get('dst') or kv.get('RemoteAddress')
-                dst_port = kv.get('DestPort') or kv.get('DestinationPort') or kv.get('dport') or kv.get('RemotePort')
-                ts = kv.get('TimeGenerated') or kv.get('EventTime') or kv.get('timestamp')
-
-                if dst_ip:
-                    conn = {
-                        'timestamp': parse_timestamp(ts) if ts else datetime.now().timestamp(),
-                        'dst_ip': dst_ip,
-                        'dst_port': int(dst_port) if dst_port and dst_port.isdigit() else 443,
-                        'bytes_sent': int(kv.get('SentBytes', 100)),
-                        'bytes_recv': int(kv.get('ReceivedBytes', 100)),
-                    }
-
-        # 5. CSV format
-        if not conn and ',' in line and not line.startswith('{'):
-            parts = [p.strip().strip('"') for p in line.split(',')]
-            if len(parts) >= 4:
-                # Try to identify columns
-                for i, p in enumerate(parts):
-                    ip, port = extract_ip_port(p)
-                    if ip:
-                        ts = None
-                        for j, q in enumerate(parts):
-                            ts = parse_timestamp(q)
-                            if ts:
-                                break
-
-                        conn = {
-                            'timestamp': ts or datetime.now().timestamp(),
-                            'dst_ip': ip,
-                            'dst_port': port or 443,
-                            'bytes_sent': 100,
-                            'bytes_recv': 100,
-                        }
-
-                        # Look for bytes
-                        for p in parts:
-                            if p.isdigit() and int(p) > 0:
-                                conn['bytes_sent'] = int(p)
-                                break
-                        break
-
-        # 6. Generic IP extraction as fallback
-        if not conn:
-            dst_ip, dst_port = extract_ip_port(line)
-            if dst_ip:
-                ts = parse_timestamp(re.search(r'[\d\-T:\.Z]+', line).group() if re.search(r'[\d\-T:\.Z]+', line) else None)
-                conn = {
-                    'timestamp': ts or datetime.now().timestamp(),
-                    'dst_ip': dst_ip,
-                    'dst_port': dst_port or 443,
-                    'bytes_sent': 100,
-                    'bytes_recv': 100,
-                }
-
-        # Validate and add
-        if conn and conn.get('dst_ip') and conn.get('timestamp'):
-            conn['dst_port'] = int(conn['dst_port'] or 443)
-            conn['bytes_sent'] = int(conn['bytes_sent'] or 0)
-            conn['bytes_recv'] = int(conn['bytes_recv'] or 0)
-            connections.append(conn)
-
-    return connections
-
-
-def analyze(input_text, file_obj, example, threshold, whitelist, blacklist):
-    """Main analysis function."""
+                    c = {'timestamp': parse_ts(p[0]), 'dst_ip': p[4] if p[4]!='-' else None, 'dst_port': int(p[5]) if p[5]!='-' else 443,
+                         'bytes_sent': int(p[9]) if p[9]!='-' else 100, 'bytes_recv': int(p[10]) if len(p)>10 and p[10]!='-' else 100}
+                except: pass
+
+        # Syslog
+        if not c:
+            m = re.match(r'^(\w{3}\s+\d+\s+\d+:\d+:\d+)\s+\S+\s+\S+:\s*(.*)$', line)
+            if m:
+                ip = re.search(r'(\d+\.\d+\.\d+\.\d+)', m.group(2))
+                if ip:
+                    c = {'timestamp': parse_ts(m.group(1)), 'dst_ip': ip.group(1), 'dst_port': 443, 'bytes_sent': 100, 'bytes_recv': 100}
+
+        # Key=value
+        if not c and '=' in line:
+            kv = dict(re.findall(r'(\w+)=(\S+)', line))
+            ip = kv.get('DestAddress') or kv.get('DestinationIp') or kv.get('dst') or kv.get('RemoteAddress')
+            if ip:
+                c = {'timestamp': parse_ts(kv.get('TimeGenerated') or kv.get('EventTime')) or datetime.now().timestamp(),
+                     'dst_ip': ip, 'dst_port': int(kv.get('DestPort', 443)), 'bytes_sent': 100, 'bytes_recv': 100}
+
+        # CSV
+        if not c and ',' in line:
+            for p in line.split(','):
+                ip = re.search(r'(\d+\.\d+\.\d+\.\d+)', p)
+                if ip:
+                    c = {'timestamp': datetime.now().timestamp(), 'dst_ip': ip.group(1), 'dst_port': 443, 'bytes_sent': 100, 'bytes_recv': 100}
+                    break
+
+        # Fallback
+        if not c:
+            ip = re.search(r'(\d+\.\d+\.\d+\.\d+)', line)
+            if ip:
+                c = {'timestamp': datetime.now().timestamp(), 'dst_ip': ip.group(1), 'dst_port': 443, 'bytes_sent': 100, 'bytes_recv': 100}
+
+        if c and c.get('dst_ip') and c.get('timestamp'):
+            c['dst_port'] = int(c.get('dst_port') or 443)
+            c['bytes_sent'] = int(c.get('bytes_sent') or 100)
+            c['bytes_recv'] = int(c.get('bytes_recv') or 100)
+            conns.append(c)
+    return conns
+
+
+def analyze(text, file, example, thresh, wl, bl):
     try:
-        # Reset lists
         sentinel.whitelist_ips = set()
         sentinel.blacklist_ips = set()
-
-        if whitelist:
-            sentinel.add_whitelist(ips=[ip.strip() for ip in whitelist.split(',') if ip.strip()])
-        if blacklist:
-            sentinel.add_blacklist(ips=[ip.strip() for ip in blacklist.split(',') if ip.strip()])
-
-        # Get connections
-        connections = []
-
-        if file_obj:
-            content = file_obj.decode('utf-8') if isinstance(file_obj, bytes) else open(file_obj, 'r').read()
-            connections = parse_logs(content)
-            if not connections:
-                try:
-                    connections = json.loads(content)
-                except:
-                    pass
-
-        if not connections and example and example in EXAMPLES:
-            connections = EXAMPLES[example]
-
-        if not connections and input_text:
-            connections = parse_logs(input_text)
-            if not connections:
-                try:
-                    connections = json.loads(input_text)
-                except:
-                    pass
-
-        if not connections:
-            return "⚠️ No valid connections found. Check input format."
-
-        if len(connections) < 3:
-            return f"⚠️ Need 3+ connections (found {len(connections)})"
-
-        # Analyze
-        result = sentinel.analyze(connections, threshold=threshold)
-
-        # Format output
-        if result.is_c2:
-            out = f"### 🚨 C2 DETECTED\n**Type:** {result.c2_type}\n\n"
-        else:
-            out = "### ✅ No C2 Detected\n\n"
-
-        out += f"**Probability:** {result.c2_probability:.0%} · **Confidence:** {result.confidence:.0%} · **Connections:** {len(connections)}\n\n"
-
-        if result.matched_legitimate_pattern:
-            out += f"**Pattern:** {result.matched_legitimate_pattern}\n\n"
-
-        if result.risk_factors:
-            out += "**Risk:** " + " · ".join(result.risk_factors[:3]) + "\n\n"
-
-        if result.recommendations and result.is_c2:
-            out += "**Action:** " + result.recommendations[0] + "\n"
-
+        if wl: sentinel.add_whitelist(ips=[x.strip() for x in wl.split(',') if x.strip()])
+        if bl: sentinel.add_blacklist(ips=[x.strip() for x in bl.split(',') if x.strip()])
+
+        conns = []
+        if file:
+            content = file.decode('utf-8') if isinstance(file, bytes) else open(file).read()
+            conns = parse_logs(content)
+            if not conns:
+                try: conns = json.loads(content)
+                except: pass
+        if not conns and example in EXAMPLES:
+            conns = EXAMPLES[example]
+        if not conns and text:
+            conns = parse_logs(text)
+            if not conns:
+                try: conns = json.loads(text)
+                except: pass
+
+        if not conns: return "No valid connections found"
+        if len(conns) < 3: return f"Need 3+ connections (found {len(conns)})"
+
+        r = sentinel.analyze(conns, threshold=thresh)
+
+        out = f"### {'C2 DETECTED: '+r.c2_type if r.is_c2 else 'No C2 Detected'}\n\n"
+        out += f"**Prob:** {r.c2_probability:.0%} | **Conf:** {r.confidence:.0%} | **N:** {len(conns)}\n\n"
+        if r.matched_legitimate_pattern: out += f"**Pattern:** {r.matched_legitimate_pattern}\n"
+        if r.risk_factors: out += "**Risk:** " + ", ".join(r.risk_factors[:3]) + "\n"
         return out
-
     except Exception as e:
-        return f"⚠️ Error: {str(e)}"
-
+        return f"Error: {e}"
 
-# UI
-with gr.Blocks(title="C2Sentinel", theme=gr.themes.Soft(), css="""
-    .container { max-width: 900px; margin: auto; }
-    footer { display: none !important; }
-""") as demo:
 
-    gr.Markdown("# 🛡️ C2Sentinel\nDetect C2 beacon patterns in network logs")
+with gr.Blocks(title="C2Sentinel") as demo:
+    gr.Markdown("## C2Sentinel - Beacon Detection\n[Docs](https://huggingface.co/danielostrow/c2sentinel)")
 
     with gr.Row():
-        with gr.Column(scale=3):
-            file_input = gr.File(label="Upload Log File", file_types=[".json", ".log", ".txt", ".csv", ".evtx"], type="binary")
-            text_input = gr.Textbox(label="Or Paste Log Data", lines=6, placeholder="JSON, syslog, Zeek, CSV, Windows events...")
-            example_select = gr.Radio(choices=list(EXAMPLES.keys()), label="Or Use Example", value=None)
-
         with gr.Column(scale=2):
-            threshold = gr.Slider(0.3, 0.8, 0.5, step=0.1, label="Sensitivity", info="Lower = more alerts")
-            whitelist = gr.Textbox(label="Whitelist IPs", placeholder="8.8.8.8, 1.1.1.1", lines=1)
-            blacklist = gr.Textbox(label="Blacklist IPs", placeholder="10.10.10.10", lines=1)
-            btn = gr.Button("Analyze", variant="primary", size="lg")
-
-    output = gr.Markdown()
-
-    gr.Markdown("""
----
-**Formats:** JSON, Zeek conn.log, syslog, Graylog, Windows Event, CSV
-**Docs:** [Model](https://huggingface.co/danielostrow/c2sentinel) · [API](https://huggingface.co/danielostrow/c2sentinel/blob/main/API_REFERENCE.md) · [neuralintellect.com](https://neuralintellect.com)
-    """)
-
-    btn.click(analyze, [text_input, file_input, example_select, threshold, whitelist, blacklist], output)
-    example_select.change(lambda x: "", [example_select], [text_input])
-
-if __name__ == "__main__":
-    demo.launch(server_name="0.0.0.0", server_port=7860)
+            file_in = gr.File(label="Log File", file_types=[".json",".log",".txt",".csv"], type="binary")
+            text_in = gr.Textbox(label="Or Paste Logs", lines=5)
+            ex_in = gr.Dropdown(choices=[""] + list(EXAMPLES.keys()), value="", label="Example")
+        with gr.Column(scale=1):
+            thresh = gr.Slider(0.3, 0.8, 0.5, label="Threshold")
+            wl = gr.Textbox(label="Whitelist", placeholder="8.8.8.8")
+            bl = gr.Textbox(label="Blacklist", placeholder="10.10.10.10")
+            btn = gr.Button("Analyze", variant="primary")
+
+    out = gr.Markdown()
+    btn.click(analyze, [text_in, file_in, ex_in, thresh, wl, bl], out)
+
+demo.launch(server_name="0.0.0.0", server_port=7860)