Update commercial_stats.py

commercial_stats.py

@@ -23,11 +23,21 @@ HTTP_WORKERS = int(os.environ.get("COMM_HTTP_WORKERS", "10"))
 BATCH_SIZE   = int(os.environ.get("COMM_SCAN_BATCH", "50"))  # reduced from 150
 SAMPLE_CAP   = 200
 
-# ---------------- helpers ----------------
+# ---------------- helpers: light HTML -> text ----------------
 _TAG_RE = re.compile(r"<[^>]+>")
 def _html_to_text(s: str) -> str:
     return _TAG_RE.sub(" ", s).strip()
 
+# ---------------- helpers: fast SPDX grep on plain text ----------------
+_SPDX_RE = re.compile(
+    r"\b(AGPL-?3\.0|AGPL|GPL-?2\.0|GPL-?3\.0|GPL|LGPL-?2\.1|LGPL-?3\.0|LGPL|"
+    r"MPL-?2\.0|BSD-2-CLAUSE|BSD-3-CLAUSE|MIT|APACHE-?2\.0|CC0-?1\.0|"
+    r"CC-BY-?4\.0|CC-BY-SA-?4\.0|CC-BY-NC-?4\.0|CC-BY-ND-?4\.0)\b",
+    re.IGNORECASE
+)
+def _spdx_grep(text: str) -> List[str]:
+    return sorted({m.group(0).upper().replace(" ", "") for m in _SPDX_RE.finditer(text)})
+
 # ---------------- in-memory state ----------------
 _state_lock = threading.Lock()
 _state: Dict[str, Any] = {
@@ -67,7 +77,6 @@ def _snapshot() -> Dict[str, Any]:
         done = _state["models_seen"]
         progress_models = (100.0 * done / tot) if tot else 0.0
 
-        # Copy shallow
         snap = {
             "phase": _state["phase"],
             "started": _state["started"],
@@ -131,8 +140,13 @@ def _run_scancode_dir(input_dir: str, output_json: str) -> Dict[str, List[str]]:
 
 def _fetch_url(session: requests.Session, url: str) -> Optional[str]:
     try:
-        r = session.get(url, timeout=HTTP_TIMEOUT)
-        if r.status_code == 200 and r.text:
+        r = session.get(
+            url,
+            headers={"User-Agent": "CommScan/1.0 (+hf.space)"},
+            timeout=HTTP_TIMEOUT,
+            allow_redirects=True,
+        )
+        if 200 <= r.status_code < 400 and r.text:
             return r.text
     except Exception:
         pass
@@ -176,7 +190,7 @@ def _scan_thread():
         for i, m in enumerate(models):
             mid = f"{m.get('owner') or m.get('username') or ''}/{m.get('name') or ''}".strip("/")
             label, url = license_label_and_url(m)
-            records.append({"mid": mid, "label": label, "url": url, "spdx": None})  # spdx None until known
+            records.append({"mid": mid, "label": label, "url": url, "spdx": None})
 
             if url:
                 url_to_models.setdefault(url, []).append(i)
@@ -209,7 +223,7 @@ def _scan_thread():
 
         # Assign unknowns for fetch failures; prepare text dedupe
         hash_to_urls: Dict[str, List[str]] = {}
-        text_hash_to_text: Dict[str, str] = {}
+        text_hash_to_text: Dict[str, Optional[str]] = {}
         for u, idxs in url_to_models.items():
             text = url_to_text.get(u)
             if not text:
@@ -226,15 +240,23 @@ def _scan_thread():
             if h not in text_hash_to_text:
                 text_hash_to_text[h] = text
 
-        # Batch scan unique texts with ScanCode
-        hashes = list(text_hash_to_text.keys())
+        # Fast-path SPDX detection by regex; mark handled hashes
+        hash_to_spdx: Dict[str, List[str]] = {}
+        for h, raw in list(text_hash_to_text.items()):
+            if raw is None:
+                continue
+            spdx = _spdx_grep(_html_to_text(raw))
+            if spdx:
+                hash_to_spdx[h] = spdx
+                text_hash_to_text[h] = None  # already satisfied
+
+        # Batch scan remaining unique texts with ScanCode
+        hashes = [k for k, v in text_hash_to_text.items() if v is not None]
         with _state_lock:
             _state["phase"] = "scan"
             _state["texts_total"] = len(hashes)
 
-        hash_to_spdx: Dict[str, List[str]] = {}
         if hashes:
-            # process in chunks to limit runtime per scancode invocation
             for i in range(0, len(hashes), BATCH_SIZE):
                 chunk = hashes[i:i+BATCH_SIZE]
                 with tempfile.TemporaryDirectory() as td:
@@ -245,7 +267,7 @@ def _scan_thread():
                     # write each text as file "<hash>.txt"
                     for h in chunk:
                         with open(os.path.join(in_dir, f"{h}.txt"), "w", encoding="utf-8") as f:
-                            f.write(_html_to_text(text_hash_to_text[h]))
+                            f.write(_html_to_text(text_hash_to_text[h]))  # strip HTML
 
                     # run scancode once over the directory
                     by_file = _run_scancode_dir(in_dir, out_json)
@@ -253,8 +275,8 @@ def _scan_thread():
                     # collect results
                     for relpath, spdx_ids in by_file.items():
                         base = os.path.basename(relpath)
-                        h = base.split(".")[0]
-                        hash_to_spdx[h] = spdx_ids
+                        hh = base.split(".")[0]
+                        hash_to_spdx[hh] = spdx_ids
 
                 with _state_lock:
                     _state["texts_scanned"] += len(chunk)
@@ -271,7 +293,6 @@ def _scan_thread():
             spdx_ids = hash_to_spdx.get(h, [])
 
             b = _bucket_spdx(spdx_ids)
-            # Update global tallies once per model
             for i in idxs:
                 rec = records[i]
                 rec["spdx"] = spdx_ids