Upload scripts/analyze_class_dist.py

scripts/analyze_class_dist.py

@@ -0,0 +1,84 @@
+"""Analyze TE family distribution from gzipped FASTA headers only (no sequences)."""
+import gzip, sys, os, json
+from collections import Counter
+from huggingface_hub import hf_hub_download
+
+print("Downloading te_rmsk_hg38_fa.gz...")
+p = hf_hub_download("vedatonuryilmaz/te_hg38", "te_rmsk_hg38_fa.gz",
+                      repo_type="dataset", token=True)
+print(f"Downloaded to: {p}")
+
+rClass = Counter()
+rFamily = Counter()
+total, nontarget = 0, 0
+NON_TARGET = {"Simple_repeat", "Low_complexity", "Unknown", "Other", "rRNA", "tRNA", "snRNA", "scRNA", "srpRNA", "RNA", "RC", "Satellite", "ARTEFACT"}
+
+with gzip.open(p, "rt") as f:
+    for line in f:
+        if line.startswith(">"):
+            total += 1
+            parts = line[1:].strip().split("|")
+            if len(parts) >= 4:
+                cls = parts[1] if len(parts) > 1 else "Unknown"
+                fam = parts[2] if len(parts) > 2 else "Unknown"
+                if cls in NON_TARGET or fam in NON_TARGET:
+                    nontarget += 1
+                    continue
+                rClass[cls] += 1
+                rFamily[fam] += 1
+            if total % 500000 == 0:
+                print(f"  ... {total:,} records, {len(rFamily)} target families")
+
+# Results
+print(f"\n=== RESULTS ===")
+print(f"Total records: {total:,}")
+print(f"Non-TE records (excluded): {nontarget:,}")
+print(f"Target TE records: {total - nontarget:,}")
+print(f"Target families: {len(rFamily)}")
+print(f"Target classes (repClass): {len(rClass)}")
+
+print(f"\n--- repClass ---")
+for c, n in rClass.most_common():
+    print(f"  {c:20s}: {n:>10,}")
+
+print(f"\n--- Size thresholds ---")
+for t in [10, 50, 100, 500, 1000, 5000, 10000, 100000]:
+    n = sum(1 for v in rFamily.values() if v >= t)
+    s = sum(v for v in rFamily.values() if v >= t)
+    print(f"  >= {t:>6}: {n:>5} families, {s:>12,} records ({100*s/max(1,total-nontarget):.1f}%)")
+
+print(f"\n--- Top 50 families ---")
+for f, n in rFamily.most_common(50):
+    pct = 100 * n / max(1, total - nontarget)
+    print(f"  {f:40s}: {n:>9,} ({pct:.1f}%)")
+
+print(f"\n--- Tail ---")
+tail = [n for n in rFamily.values() if n < 100]
+print(f"Families with <100 samples: {len(tail)} (total: {sum(tail):,})")
+tail10 = [n for n in rFamily.values() if n < 10]
+print(f"Families with <10 samples: {len(tail10)} (total: {sum(tail10):,})")
+
+# Recommended split
+MIN = 100
+good = {f: n for f, n in rFamily.items() if n >= MIN}
+rubble = sum(n for f, n in rFamily.items() if n < MIN)
+print(f"\n--- Recommendation: min={MIN} per family ---")
+print(f"  Good families: {len(good)}")
+print(f"  Total records in good: {sum(good.values()):,}")
+print(f"  Rubble records (group as 'other'): {rubble:,}")
+print(f"  Result: {len(good) + 1} classes ({len(good)} real + 1 bug bucket)")
+
+# Save
+out = {}
+out["total_records"] = total
+out["nontarget_excluded"] = nontarget
+out["target_families"] = len(rFamily)
+out["repClass_counts"] = dict(rClass.most_common())
+out["family_counts"] = dict(rFamily.most_common())
+out["thresholds"] = {str(t): {"families": sum(1 for v in rFamily.values() if v >= t),
+                               "records": sum(v for v in rFamily.values() if v >= t)}
+                      for t in [10, 50, 100, 500, 1000, 5000]}
+out["recommended_families_min100"] = list(good.keys())
+out["recommended_num_classes"] = len(good) + 1
+json.dump(out, open("class_distribution.json", "w"), indent=2)
+print("\nSaved class_distribution.json")