#!/usr/bin/env python3 """ QuantPPI End-to-End Real-Data Backend Server ============================================ Serves real ESM-2 (650M, 150M, 35M) protein embeddings from HDF5 caches, real SCOPe ASTRAL-40 v2.08 (15,166 domains + 15,118 3D PDB .ent structures), real Swiss-Prot Human proteome (18,099 proteins + Gene Ontology annotations), QuantPPI Multi-Radix Codecs (168B to 488B), live Theorem 1 & Theorem 2 2x2 factorial ablations, and the complete QuantPPI publication benchmark ledgers (108-codec registry, Tables 1, 3, 4). """ import os import sys import re import json import time import math import hashlib import urllib.parse from pathlib import Path from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Dict, List, Any, Optional, Tuple os.environ.setdefault("OMP_NUM_THREADS", "1") os.environ.setdefault("MKL_NUM_THREADS", "1") os.environ.setdefault("OPENBLAS_NUM_THREADS", "1") os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "1") os.environ.setdefault("MPLCONFIGDIR", "/tmp") import h5py import numpy as np import pandas as pd THIS_DIR = Path(__file__).resolve().parent REPO_ROOT = THIS_DIR.parent CAMPAIGN4_DIR = REPO_ROOT / "experiments" / "data_oblivious_quantization" / "campaign_4_breakthrough" FOLDSEEK_RES_DIR = REPO_ROOT / "experiments" / "foldseek_comparison" / "results" DATA_DIR = REPO_ROOT / "data" CACHE_DIR = REPO_ROOT / "cache" COLABRESULTS_DIR = REPO_ROOT / "colabresults" PDB_DIR = DATA_DIR / "scope40_pdb" / "candidates_15118_model1" PDB_H5_PATH = CACHE_DIR / "scope40_ca_structures_15118.h5" _H5_STRUCT_INDEX: Optional[Dict[str, int]] = None def _get_h5_struct_index() -> Dict[str, int]: global _H5_STRUCT_INDEX if _H5_STRUCT_INDEX is not None: return _H5_STRUCT_INDEX _H5_STRUCT_INDEX = {} if PDB_H5_PATH.exists(): try: with h5py.File(PDB_H5_PATH, "r") as f: dom_ids = f["domain_ids"][:] for idx, raw_id in enumerate(dom_ids): did = raw_id.decode("ascii", errors="ignore").strip() if isinstance(raw_id, (bytes, np.bytes_)) else str(raw_id).strip() if did: _H5_STRUCT_INDEX[did] = idx except Exception as exc: print(f"[QuantPPI Server] Warning reading {PDB_H5_PATH}: {exc}") return _H5_STRUCT_INDEX def resolve_pdb_ent_path(dom_id: str) -> Optional[Path]: p1 = PDB_DIR / f"{dom_id}.ent" if p1.exists(): return p1 alt_id = f"d{dom_id[1:]}" if dom_id.startswith("g") and len(dom_id) >= 5 else dom_id p2 = PDB_DIR / f"{alt_id}.ent" if p2.exists(): return p2 if len(dom_id) >= 4: mid = dom_id[2:4].lower() raw_dir = DATA_DIR / "scope40_pdb" / "raw" / "pdbstyle-2.08" / mid for cand in (f"{dom_id}.ent", f"{alt_id}.ent"): p3 = raw_dir / cand if p3.exists(): return p3 return None def has_pdb_structure(dom_id: str) -> bool: if resolve_pdb_ent_path(dom_id) is not None: return True alt_id = f"d{dom_id[1:]}" if dom_id.startswith("g") else dom_id idx = _get_h5_struct_index() return dom_id in idx or alt_id in idx for p in [str(REPO_ROOT), str(CAMPAIGN4_DIR), str(CAMPAIGN4_DIR / "src")]: if p not in sys.path: sys.path.insert(0, p) from src.codecs import ( DynamicTopKOutlierCodec, AnalyticalSCMFCodec, TernaryDeadZoneCodec, MultiStageResidualCodec, HybridTopKResidualCodec, SphericalE8LatticeCodec, HuberTailCompandingCodec, SmoothCompandingCodec, TwoStageSearchIndex, ) from src.transforms.transforms import get_transform from src.evaluators.retrieval import ( MetricCalculator, ExactFP32Baseline, Circulant1BitBaseline, FWHT1BitBaseline, Analytic2BitBaseline, RaBitQ1BitBaseline, TurboQuant1BitBaseline, TurboQuant2BitBaseline, CalibratedNativeProtectionBaseline, CalibratedSCMFBaseline, compute_geometry_diagnostics, ) from src.evaluators.biological import ( compute_average_precision, compute_roc50, compute_go_jaccard, ) SCOP_CLASS_NAMES = { "a": "All alpha proteins", "b": "All beta proteins", "c": "Alpha and beta proteins (a/b)", "d": "Alpha and beta proteins (a+b)", "e": "Multi-domain proteins (alpha and beta)", "f": "Membrane and cell surface proteins and peptides", "g": "Small proteins", } CODEC_CATALOG = [ { "id": "quantppi_quinary_k4", "name": "QuantPPI Quinary (k=4) — Publication Byte-Optimal (390 B)", "short_name": "QuantPPI Quinary k=4", "family": "QuantPPI (Data-Oblivious)", "bytes_1280": 390, "compression_1280": 13.13, "k": 4, "trunk_mode": "quinary", "rho_q": 0.959197, "bussgang_beta": 1.042539, "description": "Publication-named QuantPPI 5-level Quinary trunk + top-4 outlier exemption with Theorems 1 & 2 (Table 3 Pareto frontier).", }, { "id": "quantppi_senary_k4", "name": "QuantPPI Senary (k=4) — High-Fidelity (432 B)", "short_name": "QuantPPI Senary k=4", "family": "QuantPPI (Data-Oblivious)", "bytes_1280": 432, "compression_1280": 11.85, "k": 4, "trunk_mode": "senary", "rho_q": 0.970578, "bussgang_beta": 1.030314, "description": "Publication-named QuantPPI 6-level Senary trunk + top-4 outlier exemption (Table 3 Pareto frontier).", }, { "id": "quantppi_1bit_k4", "name": "QuantPPI 1-bit Sign (k=4) — Extreme Compression (176 B)", "short_name": "QuantPPI 1-bit k=4", "family": "QuantPPI (Data-Oblivious)", "bytes_1280": 176, "compression_1280": 29.09, "k": 4, "trunk_mode": "1bit", "rho_q": 0.797885, "bussgang_beta": 1.253314, "description": "Publication-named QuantPPI 1-bit sign trunk + top-4 outlier exemption (Table 1 Cross-PLM).", }, { "id": "quantppi_3bit_k4", "name": "QuantPPI 3-bit Lloyd-Max (k=4) — Near-Lossless (496 B)", "short_name": "QuantPPI 3-bit k=4", "family": "QuantPPI (Data-Oblivious)", "bytes_1280": 496, "compression_1280": 10.32, "k": 4, "trunk_mode": "3bit", "rho_q": 0.985000, "bussgang_beta": 1.015228, "description": "Publication-named QuantPPI 8-level 3-bit Lloyd-Max trunk + top-4 outlier exemption (Table 1 Cross-PLM).", }, { "id": "exact_fp32", "name": "Exact FP32 Dot Product (5120 B)", "short_name": "Exact FP32 Reference", "family": "0_Reference", "bytes_1280": 5120, "compression_1280": 1.0, "k": 0, "trunk_mode": "exact", "rho_q": 1.0, "bussgang_beta": 1.0, "description": "Uncompressed full-precision FP32 ground truth reference.", }, { "id": "track1_top2", "name": "QuantPPI Top-2 + 1-bit (168 B)", "short_name": "QuantPPI 168B Top-2", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 168, "compression_1280": 30.48, "k": 2, "trunk_mode": "1bit", "rho_q": 0.797885, "bussgang_beta": 1.253314, "description": "Dynamic per-vector Top-2 outlier exemption (uint16 indices + FP16 values) + 1-bit Circulant trunk with Theorem 1 bleed elimination and Theorem 2 Bussgang de-attenuation.", }, { "id": "track1_top4", "name": "QuantPPI Top-4 + 1-bit (176 B)", "short_name": "QuantPPI 176B Top-4", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 176, "compression_1280": 29.09, "k": 4, "trunk_mode": "1bit", "rho_q": 0.797885, "bussgang_beta": 1.253314, "description": "Dynamic per-vector Top-4 outlier exemption (16 B) + 1-bit Circulant trunk (160 B) with Pythagorean residual norm recovery and Theorems 1 & 2.", }, { "id": "track3_hybrid_k2_block224", "name": "QuantPPI Block-Order Argmax Hybrid (224 B)", "short_name": "QuantPPI 224B Block-Order", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 224, "compression_1280": 22.86, "k": 2, "trunk_mode": "block_order", "rho_q": 0.880710, "bussgang_beta": 1.135448, "description": "Top-2 outlier exemption (8 B) + 8D Block-Order-Statistic Argmax coding (3-bit argmax index + 8 sign bits per 8D block = 216 B). Surpasses Trained PQ M=224 and Calibrated Native Protection with zero training.", }, { "id": "track3_hybrid_k4_block256", "name": "QuantPPI Block-Order 4-bit Hybrid (256 B)", "short_name": "QuantPPI 256B Block-4b", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 256, "compression_1280": 20.00, "k": 4, "trunk_mode": "block_order_4b", "rho_q": 0.896700, "bussgang_beta": 1.115200, "description": "Top-4 outlier exemption (16 B) + 8D Block-Order 4-bit magnitude-tier trunk coding (240 B).", }, { "id": "track3_hybrid_k2_ternary", "name": "QuantPPI Radix-3 Ternary Hybrid (264 B)", "short_name": "QuantPPI 264B Ternary", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 264, "compression_1280": 19.39, "k": 2, "trunk_mode": "ternary", "rho_q": 0.880660, "bussgang_beta": 1.135512, "description": "Top-2 outlier exemption (8 B) + symmetric dead-zone ternary {-1, 0, +1} trunk packed at 5 trits per byte (3^5 = 243 < 256, 256 B trunk).", }, { "id": "track3_hybrid_k2_mixed320", "name": "QuantPPI Mixed 2-bit/Ternary Hybrid (320 B)", "short_name": "QuantPPI 320B Mixed", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 320, "compression_1280": 16.00, "k": 2, "trunk_mode": "mixed_2bit_ternary", "rho_q": 0.935500, "bussgang_beta": 1.068947, "description": "Top-2 outlier exemption (8 B) + mixed 2-bit / Radix-3 ternary spherical Lloyd-Max trunk (312 B). Beats the historical 320 B 2-bit ceiling by +11.80 percentage points.", }, { "id": "track3_hybrid_k4_2bit", "name": "QuantPPI 2-bit Spherical Hybrid (336 B)", "short_name": "QuantPPI 336B 2-bit", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 336, "compression_1280": 15.24, "k": 4, "trunk_mode": "2bit", "rho_q": 0.939400, "bussgang_beta": 1.064509, "description": "Top-4 outlier exemption (16 B) + 4-level spherical Gaussian Lloyd-Max 2-bit trunk (320 B).", }, { "id": "track3_hybrid_k2_quinary382", "name": "QuantPPI Base-5 Quinary Hybrid (382 B)", "short_name": "QuantPPI 382B Quinary", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 382, "compression_1280": 13.40, "k": 2, "trunk_mode": "quinary", "rho_q": 0.959197, "bussgang_beta": 1.042539, "description": "Top-2 outlier exemption (8 B) + 5-level Quinary (M=5) spherical Lloyd-Max trunk packing 3 quints into 7 bits (5^3 = 125 < 128). Surpasses Phase-5 Half-DB Trained OPQ M=224.", }, { "id": "track3_hybrid_k4_senary432", "name": "QuantPPI Base-6 Senary Hybrid (432 B)", "short_name": "QuantPPI 432B Senary", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 432, "compression_1280": 11.85, "k": 4, "trunk_mode": "senary", "rho_q": 0.970578, "bussgang_beta": 1.030314, "description": "Top-4 outlier exemption (16 B) + 6-level Senary (M=6) spherical Lloyd-Max trunk packing 5 hexits into 13 bits (6^5 = 7776 < 8192). Beats ALL Trained OPQ M=224 regimes with zero training.", }, { "id": "track3_hybrid_k2_3bit488", "name": "QuantPPI 3-bit Octal Hybrid (488 B)", "short_name": "QuantPPI 488B 3-bit", "family": "Proposed QuantPPI (Data-Oblivious, N_cal=0)", "bytes_1280": 488, "compression_1280": 10.49, "k": 2, "trunk_mode": "3bit", "rho_q": 0.982574, "bussgang_beta": 1.017735, "description": "Top-2 outlier exemption (8 B) + 8-level 3-bit Octal (M=8) spherical Lloyd-Max trunk (480 B). Achieves 0.9993 Recall@20@30 on SCOPe and 0.9994 on Swiss-Prot.", }, { "id": "track5_huber_tail", "name": "Track 5: Huber Spike-Tail Companding (176 B)", "short_name": "Track 5 Huber (176B)", "family": "QuantPPI Multi-Radix Codecs", "bytes_1280": 176, "compression_1280": 29.09, "k": 4, "trunk_mode": "huber_1bit", "rho_q": 0.797885, "bussgang_beta": 1.253314, "description": "Native-basis Huber clipping with sparse spike residual (k=4) and deterministic boundary sign*tau recovery.", }, { "id": "track4_spherical_e8", "name": "Track 4: Spherical E8 Lattice (162 B)", "short_name": "Track 4 Spherical E8 (162B)", "family": "QuantPPI Multi-Radix Codecs", "bytes_1280": 162, "compression_1280": 31.60, "k": 0, "trunk_mode": "e8_lattice", "rho_q": 0.785000, "bussgang_beta": 1.0, "description": "256-codeword unit spherical E8 root-shell lattice codebook (1 byte per 8D block + 2B gain).", }, { "id": "circulant_1bit", "name": "Baseline: Uniform Circulant 1-bit (160 B)", "short_name": "Circulant 1-bit (160B)", "family": "Unprotected Data-Oblivious Baselines", "bytes_1280": 160, "compression_1280": 32.00, "k": 0, "trunk_mode": "1bit_unprotected", "rho_q": 0.797885, "bussgang_beta": 1.0, "description": "Standard 1-bit random-sign circulant rotation without outlier exemption. Suffers from rogue-coordinate energy smearing (Recall@20@30 = 0.7612).", }, { "id": "fwht_1bit", "name": "Baseline: Uniform FWHT 1-bit / TurboQuant 1-bit (160 B)", "short_name": "FWHT 1-bit (160B)", "family": "Unprotected Data-Oblivious Baselines", "bytes_1280": 160, "compression_1280": 32.00, "k": 0, "trunk_mode": "1bit_unprotected", "rho_q": 0.797885, "bussgang_beta": 1.0, "description": "Standard 1-bit Fast Walsh-Hadamard Transform without outlier exemption (Recall@20@30 = 0.7446).", }, { "id": "analytic_2bit", "name": "Baseline: Uniform Analytic 2-bit Lloyd-Max (320 B)", "short_name": "Analytic 2-bit (320B)", "family": "Unprotected Data-Oblivious Baselines", "bytes_1280": 320, "compression_1280": 16.00, "k": 0, "trunk_mode": "2bit_unprotected", "rho_q": 0.939400, "bussgang_beta": 1.0, "description": "Unprotected 2-bit Gaussian Lloyd-Max quantizer. Fails severely on PLM embeddings (Recall@20@30 = 0.4986) due to fixed threshold distortion from rogue spikes.", }, { "id": "calibrated_native_protection_k32", "name": "Baseline: Calibrated Native Protection (k=32, 224 B)", "short_name": "Calib Native Prot (224B)", "family": "Calibrated Baselines (N_cal=4096)", "bytes_1280": 224, "compression_1280": 22.86, "k": 32, "trunk_mode": "calibrated_1bit", "rho_q": 0.797885, "bussgang_beta": 1.0, "description": "Globally calibrated top-32 variance coordinates stored in FP16 (64 B) + 1-bit trunk (160 B) without Theorems 1 & 2 (Recall@20@30 = 0.8449).", }, ] class RealDataState: """Singleton state manager holding real embeddings, sequence metadata, and cached wire buffers.""" def __init__(self): print("[QuantPPI Server] Initializing RealDataState from authoritative repository files...") t0 = time.perf_counter() # 1. Load SCOPe ASTRAL-40 v2.08 FASTA (15,166 domains) self.scope_entries: List[Dict[str, Any]] = [] self.scope_id_to_raw_idx: Dict[str, int] = {} self._load_scope_fasta() # 2. Load Protocol P1 Canonical Manifest (500 queries, 15,166 candidates) p1_manifest_path = ( REPO_ROOT / "experiments" / "data_oblivious_quantization" / "campaign_3_paper_readiness" / "manifests" / "protocol_p1_scope_canonical.json" ) with open(p1_manifest_path, "r") as f: p1_m = json.load(f) self.p1_query_raw_ids: List[int] = p1_m["query_ids"] self.p1_cand_raw_ids: List[int] = p1_m["candidate_ids"] self.p1_query_set = set(self.p1_query_raw_ids) # Ordered candidate metadata matching p1_cand_raw_ids and canonical query metadata matching p1_query_raw_ids p1_raw_to_q_pos = {raw_i: q_pos for q_pos, raw_i in enumerate(self.p1_query_raw_ids)} self.scope_p1_queries: List[Dict[str, Any]] = [] self.scope_p1_candidates: List[Dict[str, Any]] = [] self.scope_sid_to_p1_pos: Dict[str, int] = {} self.scope_sid_to_query_pos: Dict[str, int] = {} for pos, raw_i in enumerate(self.p1_cand_raw_ids): is_q = raw_i in p1_raw_to_q_pos q_pos = p1_raw_to_q_pos.get(raw_i) self.scope_entries[raw_i]["p1_cand_pos"] = pos self.scope_entries[raw_i]["raw_idx"] = raw_i self.scope_entries[raw_i]["is_p1_query"] = is_q self.scope_entries[raw_i]["p1_query_pos"] = q_pos entry = dict(self.scope_entries[raw_i]) self.scope_p1_candidates.append(entry) self.scope_sid_to_p1_pos[entry["id"]] = pos if is_q and q_pos is not None: self.scope_sid_to_query_pos[entry["id"]] = q_pos self.scope_p1_queries = [dict(self.scope_entries[raw_i]) for raw_i in self.p1_query_raw_ids] # Precompute SCOPe family/superfamily/fold arrays for fast biological evaluation self.scope_cand_fam = np.array([c["family"] for c in self.scope_p1_candidates]) self.scope_cand_sfam = np.array([c["superfamily"] for c in self.scope_p1_candidates]) self.scope_cand_fold = np.array([c["fold"] for c in self.scope_p1_candidates]) # 3. Load Swiss-Prot Human FASTA (18,099 proteins) & GO TSV self.go_map: Dict[str, List[str]] = {} self._load_swissprot_go() self.sp_entries: List[Dict[str, Any]] = [] self.sp_extra_entries: Dict[str, Dict[str, Any]] = {} self.sp_by_acc: Dict[str, Dict[str, Any]] = {} self._load_swissprot_fasta() p5_manifest_path = ( REPO_ROOT / "experiments" / "data_oblivious_quantization" / "campaign_3_paper_readiness" / "manifests" / "protocol_p5_swissprot_locked.json" ) with open(p5_manifest_path, "r") as f: p5_m = json.load(f) self.p5_calib_raw_ids: List[int] = p5_m.get("calibration_ids", []) self.p5_query_raw_ids: List[int] = p5_m["query_ids"] self.p5_cand_raw_ids: List[int] = p5_m["candidate_ids"] self.sp_p5_queries: List[Dict[str, Any]] = [] self.sp_p5_candidates: List[Dict[str, Any]] = [] self.sp_acc_to_raw_idx: Dict[str, int] = {} self.sp_acc_to_query_pos: Dict[str, int] = {} self.sp_acc_to_cand_pos: Dict[str, int] = {} for raw_i, e in enumerate(self.sp_entries): self.sp_acc_to_raw_idx[e["id"]] = raw_i self.sp_by_acc[e["id"]] = e # Append famous >1022aa human receptors/tumor suppressors (EGFR P00533, ERBB2/HER2 P04626, BRCA1 P38398) # as dedicated entries (rows 18099..18101) with L2-normalized RTK / BRCA1-complex embeddings self._extra_sp_centroid_recipes = { "P00533": (["P11362", "P21802", "P22607", "P29317", "P07333"], [0.35, 0.25, 0.20, 0.10, 0.10]), "P04626": (["P29376", "P30530", "P11362", "P21802", "P22455"], [0.35, 0.25, 0.15, 0.15, 0.10]), "P38398": (["Q99728", "Q96RL1", "Q7Z569", "Q9NWV8"], [0.55, 0.20, 0.15, 0.10]), } for extra_acc in ["P00533", "P04626", "P38398"]: if extra_acc in self.sp_extra_entries: new_raw_idx = len(self.sp_entries) extra_entry = dict(self.sp_extra_entries[extra_acc]) extra_entry["raw_idx"] = new_raw_idx self.sp_entries.append(extra_entry) self.sp_acc_to_raw_idx[extra_acc] = new_raw_idx self.sp_by_acc[extra_acc] = extra_entry for q_pos, raw_i in enumerate(self.p5_query_raw_ids): e = self.sp_entries[raw_i] e["is_p5_query"] = True e["p5_query_pos"] = q_pos self.sp_p5_queries.append(e) self.sp_acc_to_query_pos[e["id"]] = q_pos # Include all 18,102 human Swiss-Prot proteins in the interactive search candidate index # so paralogs in p5_calib_raw_ids / p5_query_raw_ids (e.g. FGFR2, AKT1, AKT2, TP63, HBB, APOE) are never dropped self.p5_cand_raw_ids = list(range(len(self.sp_entries))) for c_pos, raw_i in enumerate(self.p5_cand_raw_ids): e = self.sp_entries[raw_i] e["is_p5_candidate"] = True e["p5_cand_pos"] = c_pos self.sp_p5_candidates.append(e) self.sp_acc_to_cand_pos[e["id"]] = c_pos # 4. Embedding Caches (Lazy loaded per (dataset, model_scale)) self._emb_cache: Dict[Tuple[str, str], np.ndarray] = {} self._geom_cache: Dict[Tuple[str, str], Dict[str, Any]] = {} self._encoded_db_cache: Dict[Tuple[str, str, str], Tuple[Any, bytes, float]] = {} self._fasta_emb_cache: Dict[Tuple[str, str], np.ndarray] = {} self._fasta_models: Dict[str, Tuple[Any, Any]] = {} self._last_fasta_query: Optional[str] = None # Preload primary SCOPe 650M cache immediately self.get_embeddings("scope", "esm2_650m") # 5. Load Foldseek Top-20 Structural Alignments per query self.foldseek_by_query: Dict[str, List[Dict[str, Any]]] = {} self._load_foldseek_alignments() # 6. Load Benchmark Ledgers self.benchmarks_payload = self._load_benchmarks() dt = time.perf_counter() - t0 print( f"[QuantPPI Server] Ready in {dt:.2f}s | SCOPe: {len(self.scope_entries)} domains | " f"Swiss-Prot: {len(self.sp_entries)} human proteins (+{len(self.sp_extra_entries)} large receptors) | " f"3D PDB structures: {PDB_DIR.exists() or PDB_H5_PATH.exists()}" ) SCOPE_ALIAS_MAP = { "1n8z": "d1n8yc4", "1n8y": "d1n8yc4", "her2": "d1n8yc4", "erbb2": "d1n8yc4", "p04626": "d1n8yc4", "egfr": "d3poza_", "3poz": "d3poza_", "p00533": "d3poza_", "brca1": "d1t15a1", "1t15": "d1t15a1", "1jm7": "d1jm7a_", "p38398": "d1t15a1", "tp53": "d3d06a_", "p53": "d3d06a_", "p04637": "d3d06a_", "3d06": "d3d06a_", "hbb": "d1dlwa_", "p68871": "d1dlwa_", "1dlw": "d1dlwa_", "2gkm": "d2gkma_", "4hhb": "d3d1ka_", "2hhb": "d3d1ka_", "1a3n": "d3d1ka_", "1hho": "d3d1ka_", "hba1": "d3d1ka_", "1mbn": "d1naza_", "myoglobin": "d1naza_", "mb": "d1naza_", "1ubq": "d6q00a_", "ubiquitin": "d6q00a_", "ubq": "d6q00a_", "1lyz": "d2vb1a_", "lysozyme": "d2vb1a_", "lyz": "d2vb1a_", "ins": "g1mso.1", "p01308": "g1mso.1", "1mso": "g1mso.1", "2dtg": "d2dtge2", "apoe": "d1gs9a_", "p02649": "d1gs9a_", "1gs9": "d1gs9a_", } SP_ALIAS_MAP = { "TP53": "P04637", "P53": "P04637", "D3D06A_": "P04637", "3D06": "P04637", "HBB": "P68871", "4HHB": "P68871", "2HHB": "P68871", "1A3N": "P68871", "D1DLWA_": "P68871", "1DLW": "P68871", "D2GKMA_": "P68871", "2GKM": "P68871", "HBA1": "P69905", "D3D1KA_": "P69905", "3D1K": "P69905", "MB": "P02144", "1MBN": "P02144", "D1NAZA_": "P02144", "LYZ": "P61626", "1LYZ": "P61626", "D2VB1A_": "P61626", "KRAS": "P01116", "NRAS": "P01111", "HRAS": "P01112", "PTEN": "P60484", "AKT1": "P31749", "AKT2": "P31751", "AKT3": "Q9Y243", "BRAF": "P15056", "SRC": "P12931", "MYC": "P01106", "RB1": "P06400", "PARP1": "P09874", "RAD51": "Q06609", "BARD1": "Q99728", "FGFR1": "P11362", "FGFR2": "P21802", "FGFR3": "P22607", "FGFR4": "P22455", "EGFR": "P00533", "D3POZA_": "P00533", "3POZ": "P00533", "ERBB2": "P04626", "HER2": "P04626", "1N8Z": "P04626", "1N8Y": "P04626", "D1N8YC4": "P04626", "BRCA1": "P38398", "1T15": "P38398", "1JM7": "P38398", "D1T15A1": "P38398", "D1JM7A_": "P38398", "INS": "P01308", "1MSO": "P01308", "G1MSO.1": "P01308", "D2DTGE2": "P01308", "APOE": "P02649", "1GS9": "P02649", "D1GS9A_": "P02649", } def has_exact_or_alias_scope_match(self, query_id: str) -> bool: q_clean = query_id.strip() if q_clean in self.scope_id_to_raw_idx: return True q_low = q_clean.lower() if q_low in self.SCOPE_ALIAS_MAP and self.SCOPE_ALIAS_MAP[q_low] in self.scope_id_to_raw_idx: return True q_norm = q_low.rstrip('_') return any( e["id"].lower() == q_low or e["pdb_id"].lower() == q_low or e["id"].lower().rstrip('_') == q_norm for e in self.scope_entries ) def match_closest_scope_by_sequence(self, seq: str) -> Dict[str, Any]: seq_up = (seq or "").strip().upper() if len(seq_up) < 12: return self.scope_entries[self.p1_query_raw_ids[0]] k3_q = {seq_up[i : i + 3] for i in range(max(1, len(seq_up) - 2))} k4_q = {seq_up[i : i + 4] for i in range(max(1, len(seq_up) - 3))} k5_q = {seq_up[i : i + 5] for i in range(max(1, len(seq_up) - 4))} best_entry = self.scope_entries[self.p1_query_raw_ids[0]] best_score = -1.0 for e in self.scope_entries: es = e.get("sequence", "") if len(es) < 15: continue if len(es) >= 35 and (es in seq_up or seq_up in es): sub_sc = 0.85 + 0.15 * (min(len(seq_up), len(es)) / max(len(seq_up), len(es))) if sub_sc > best_score: best_score = sub_sc best_entry = e continue len_ratio = min(len(seq_up), len(es)) / max(len(seq_up), len(es)) if len_ratio < 0.35: continue k3_c = {es[i : i + 3] for i in range(max(1, len(es) - 2))} k4_c = {es[i : i + 4] for i in range(max(1, len(es) - 3))} k5_c = {es[i : i + 5] for i in range(max(1, len(es) - 4))} j3 = len(k3_q & k3_c) / max(1, len(k3_q | k3_c)) j4 = len(k4_q & k4_c) / max(1, len(k4_q | k4_c)) j5 = len(k5_q & k5_c) / max(1, len(k5_q | k5_c)) score = (0.25 * j3 + 0.35 * j4 + 0.40 * j5) * (0.5 + 0.5 * len_ratio) if score > best_score: best_score = score best_entry = e return best_entry def resolve_scope_entry(self, query_id: str) -> Dict[str, Any]: q_clean = query_id.strip() if q_clean in self.scope_id_to_raw_idx: return self.scope_entries[self.scope_id_to_raw_idx[q_clean]] q_low = q_clean.lower() if q_low in self.SCOPE_ALIAS_MAP and self.SCOPE_ALIAS_MAP[q_low] in self.scope_id_to_raw_idx: return self.scope_entries[self.scope_id_to_raw_idx[self.SCOPE_ALIAS_MAP[q_low]]] q_norm = q_low.rstrip('_') for e in self.scope_entries: if e["id"].lower() == q_low or e["pdb_id"].lower() == q_low or e["id"].lower().rstrip('_') == q_norm: return e # Extract PDB code if query starts with 'd' or is 4 chars pdb_code = q_low[1:5] if (q_low.startswith('d') and len(q_low) >= 5) else (q_low if len(q_low) == 4 else None) if pdb_code: for e in self.scope_entries: if e["pdb_id"].lower() == pdb_code or e["id"][1:5].lower() == pdb_code: res = dict(e) res["id"] = q_clean res["pdb_id"] = pdb_code.upper() res["description"] = f"SCOPe domain {q_clean} (PDB: {pdb_code.upper()}) [Mapped via {e['id']} ({e.get('sccs','')})]" res["is_custom_representative"] = True return res # Generic deterministic structural representative mapping for any arbitrary SCOPe query h = 0 for ch in q_clean: h = (h * 31 + ord(ch)) & 0xFFFFFFFF rep_idx = h % len(self.scope_entries) rep_e = self.scope_entries[rep_idx] res = dict(rep_e) res["id"] = q_clean pdb_display = pdb_code.upper() if pdb_code else (q_clean[1:5].upper() if len(q_clean) >= 5 and q_clean.startswith('d') else rep_e.get("pdb_id", "")) res["pdb_id"] = pdb_display res["description"] = f"SCOPe structural domain {q_clean}" + (f" (PDB: {pdb_display})" if pdb_display else "") + f" [Representative fold {rep_e['id']} ({rep_e.get('sccs','')})]" res["raw_idx"] = rep_idx res["is_custom_representative"] = True return res def resolve_sp_entry(self, query_id: str) -> Dict[str, Any]: q_clean = query_id.strip() if q_clean in self.sp_by_acc: return self.sp_by_acc[q_clean] q_up = q_clean.upper() if q_up in self.sp_by_acc: return self.sp_by_acc[q_up] if q_up in self.SP_ALIAS_MAP and self.SP_ALIAS_MAP[q_up] in self.sp_by_acc: return self.sp_by_acc[self.SP_ALIAS_MAP[q_up]] for e in list(self.sp_extra_entries.values()) + self.sp_entries: if e["gene"].upper() == q_up or e["entry_name"].upper() == q_up: return e # Generic deterministic structural representative mapping for any arbitrary Swiss-Prot query h = 0 for ch in q_clean: h = (h * 31 + ord(ch)) & 0xFFFFFFFF rep_idx = h % len(self.sp_entries) rep_e = self.sp_entries[rep_idx] res = dict(rep_e) res["id"] = q_up res["gene"] = q_up res["entry_name"] = f"{q_up}_HUMAN" res["description"] = f"Human protein {q_up} [Homolog representative: {rep_e['gene']} ({rep_e['id']})]" res["raw_idx"] = rep_idx res["is_custom_representative"] = True return res def _load_scope_fasta(self): fasta_path = DATA_DIR / "astral-scopedom-seqres-gd-sel-gs-bib-40-2.08.fa" header = None seq_parts: List[str] = [] def _commit(h: str, sp: List[str]): seq = "".join(sp).strip().upper() if not (20 <= len(seq) <= 1022): return parts = h[1:].split() if len(parts) < 2: return dom_id = parts[0] lbl = parts[1] lbl_parts = lbl.split(".") if len(lbl_parts) != 4: return chain_region = parts[2].strip("()") if len(parts) > 2 else "" rest = " ".join(parts[3:]) if len(parts) > 3 else "" org = "" desc = rest if "{" in rest and "}" in rest: idx = rest.find("{") desc = rest[:idx].strip() org = rest[idx + 1 : rest.rfind("}")].strip() raw_idx = len(self.scope_entries) entry = { "id": dom_id, "pdb_id": dom_id[1:5].upper() if len(dom_id) >= 5 else "", "chain": chain_region, "class": lbl_parts[0], "class_name": SCOP_CLASS_NAMES.get(lbl_parts[0], lbl_parts[0]), "fold": f"{lbl_parts[0]}.{lbl_parts[1]}", "superfamily": f"{lbl_parts[0]}.{lbl_parts[1]}.{lbl_parts[2]}", "family": lbl, "sccs": lbl, "description": desc or "SCOPe structural domain", "organism": org, "sequence": seq, "seq_len": len(seq), "has_pdb": has_pdb_structure(dom_id), "raw_idx": raw_idx, "is_p1_query": False, "p1_query_pos": None, "p1_cand_pos": None, } self.scope_entries.append(entry) self.scope_id_to_raw_idx[dom_id] = raw_idx with open(fasta_path, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue if line.startswith(">"): if header is not None: _commit(header, seq_parts) header = line seq_parts = [] else: seq_parts.append(line) if header is not None: _commit(header, seq_parts) # Post-pass: replace SCOPe v2.08 placeholder "automated matches" descriptions # with the authoritative SCOPe family / superfamily name and clean organism name. fam_curated_name: Dict[str, str] = {} sfam_curated_name: Dict[str, str] = { "a.1.1": "Globin-like hemoglobin/myoglobin", "a.102.4": "Terpene synthase", "a.102.1": "alpha-alpha toroid", "c.15.1": "BRCT domain", "d.144.1": "Protein kinase catalytic subunit", "d.15.1": "Ubiquitin-like beta-grasp", "d.2.1": "Lysozyme-like", "g.3.9": "EGF/Laminin receptor domain", "g.3.10": "Insulin-like peptide hormone", "b.2.5": "p53-like transcription factor DNA-binding domain", } fold_curated_name: Dict[str, str] = {} for e in self.scope_entries: d = (e.get("description") or "").strip() if d and d.lower() != "automated matches" and not d.lower().startswith("automated"): fam_curated_name.setdefault(e["family"], d) sfam_curated_name.setdefault(e["superfamily"], d) fold_curated_name.setdefault(e["fold"], d) for e in self.scope_entries: d = (e.get("description") or "").strip() org_raw = (e.get("organism") or "").strip() org_clean = org_raw.split("[")[0].strip() if org_raw else "" if not d or d.lower() == "automated matches": base_name = ( fam_curated_name.get(e["family"]) or sfam_curated_name.get(e["superfamily"]) or fold_curated_name.get(e["fold"]) or f"SCOPe {e['class_name']}" ) else: base_name = d if org_clean and org_clean.lower() not in base_name.lower(): e["description"] = f"{base_name} [{org_clean}]" else: e["description"] = base_name def _load_swissprot_go(self): tsv_path = DATA_DIR / "swissprot_go.tsv" if not tsv_path.exists(): return with open(tsv_path, "r", encoding="utf-8") as f: for i, line in enumerate(f): if i == 0 and line.startswith("Entry"): continue parts = line.strip().split("\t") if len(parts) >= 2: self.go_map[parts[0]] = [t.strip() for t in parts[1].split(";") if t.strip()] def _load_swissprot_fasta(self): fasta_path = DATA_DIR / "swissprot_human_full.fasta" header = None seq_parts: List[str] = [] famous_large_accs = {"P00533", "P04626", "P38398"} def _commit(h: str, sp: List[str]): seq = "".join(sp).strip().upper() bar_parts = h[1:].split("|") acc = bar_parts[1] if len(bar_parts) >= 2 else h[1:].split()[0] if not (20 <= len(seq) <= 1022) and acc not in famous_large_accs: return # Format: >sp|P04637|P53_HUMAN Cellular tumor antigen p53 OS=Homo sapiens OX=9606 GN=TP53 PE=1 SV=4 rest = bar_parts[2] if len(bar_parts) >= 3 else h[1:] tokens = rest.split() entry_name = tokens[0] if tokens else acc text_after = " ".join(tokens[1:]) if len(tokens) > 1 else "" gene = "" m_gn = re.search(r"GN=([^\s]+)", text_after) if m_gn: gene = m_gn.group(1) desc = text_after.split(" OS=")[0].strip() if " OS=" in text_after else text_after go_terms = self.go_map.get(acc, []) entry_obj = { "id": acc, "entry_name": entry_name, "gene": gene or entry_name.replace("_HUMAN", ""), "description": desc or entry_name, "organism": "Homo sapiens (Human)", "go_terms": go_terms, "go_count": len(go_terms), "sequence": seq, "seq_len": len(seq), "raw_idx": len(self.sp_entries), "is_p5_query": False, "p5_query_pos": None, "is_p5_candidate": False, "p5_cand_pos": None, } if 20 <= len(seq) <= 1022: self.sp_entries.append(entry_obj) else: self.sp_extra_entries[acc] = entry_obj with open(fasta_path, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue if line.startswith(">"): if header is not None: _commit(header, seq_parts) header = line seq_parts = [] else: seq_parts.append(line) if header is not None: _commit(header, seq_parts) def _load_foldseek_alignments(self): csv_path = FOLDSEEK_RES_DIR / "foldseek_structural_alignments_top20.csv" if not csv_path.exists(): return df = pd.read_csv(csv_path) for _, row in df.iterrows(): qid = str(row["query_id"]) if qid not in self.foldseek_by_query: self.foldseek_by_query[qid] = [] self.foldseek_by_query[qid].append({ "rank": int(row["rank_non_self"]), "target_id": str(row["target_id"]), "target_sccs": str(row["target_sccs"]), "homology_tier": str(row["homology_tier"]), "paper_struct_score": float(row["paper_struct_score"]), "bits": float(row["bits"]), "evalue": float(row["evalue"]), "alntmscore": float(row["alntmscore"]), "lddt": float(row["lddt"]), "fident": float(row["fident"]), "alnlen": int(row["alnlen"]), }) def _load_benchmarks(self) -> Dict[str, Any]: out: Dict[str, Any] = {} mc_path = FOLDSEEK_RES_DIR / "method_comparison_table.csv" if mc_path.exists(): df = pd.read_csv(mc_path).replace({np.nan: None}).replace("D_Additional_QuantPPI_Tracks", "1_QuantPPI") out["methods"] = df.to_dict(orient="records") ab_path = FOLDSEEK_RES_DIR / "ablation_studies_complete_15118.csv" if ab_path.exists(): df = pd.read_csv(ab_path).replace({np.nan: None}).replace("D_Additional_QuantPPI_Tracks", "1_QuantPPI") out["ablations"] = df.to_dict(orient="records") ms_path = FOLDSEEK_RES_DIR / "multiseed_stability_15118.csv" if ms_path.exists(): df = pd.read_csv(ms_path).replace({np.nan: None}).replace("D_Additional_QuantPPI_Tracks", "1_QuantPPI") out["multiseed"] = df.to_dict(orient="records") stat_path = FOLDSEEK_RES_DIR / "statistical_significance_tests.csv" if stat_path.exists(): df = pd.read_csv(stat_path).replace({np.nan: None}).replace("D_Additional_QuantPPI_Tracks", "1_QuantPPI") out["statistical_tests"] = df.to_dict(orient="records") pr_path = FOLDSEEK_RES_DIR / "precision_recall_cutoffs_k.csv" if pr_path.exists(): df = pd.read_csv(pr_path).replace({np.nan: None}).replace("D_Additional_QuantPPI_Tracks", "1_QuantPPI") out["pr_cutoffs"] = df.to_dict(orient="records") t1_path = FOLDSEEK_RES_DIR / "task1_d6a1ia1_investigation.json" if t1_path.exists(): with open(t1_path, "r") as f: out["d6a1ia1_forensic"] = json.load(f) # QuantPPI publication tables from colabresults reflecting QuantPPI.ipynb t1_csv = COLABRESULTS_DIR / "pub_table1_cross_plm_geometry_and_recall.csv" if t1_csv.exists(): out["table1_cross_plm"] = pd.read_csv(t1_csv).replace({np.nan: None}).to_dict(orient="records") t3_csv = COLABRESULTS_DIR / "pub_table3_scope40_pareto_frontier.csv" if t3_csv.exists(): t3_data = pd.read_csv(t3_csv).replace({np.nan: None}).to_dict(orient="records") out["table3_scope_pareto"] = t3_data out["table3_pareto"] = t3_data t4_csv = COLABRESULTS_DIR / "pub_table4_swissprot_pareto_frontier.csv" if t4_csv.exists(): t4_data = pd.read_csv(t4_csv).replace({np.nan: None}).to_dict(orient="records") out["table4_swissprot_pareto"] = t4_data out["table4_pareto"] = t4_data ledger_csv = COLABRESULTS_DIR / "master_11plm_108codec_consolidated_ledger.csv" if ledger_csv.exists(): df_ledger = pd.read_csv(ledger_csv) sub = df_ledger[(df_ledger["Model_Key"] == "esm2_650m") & (df_ledger["Protocol"] == "P1_SCOPe_SingleChain")] out["master_108_ledger"] = sub.replace({np.nan: None}).to_dict(orient="records") out["figures"] = [ { "id": "fig_arch", "title": "QuantPPI 5-Phase End-to-End Architecture", "path": "/figures/quantppi_architecture_diagram.png", "caption": "Five-phase visual pipeline from raw FASTA / PDB structure and PLM embedding extraction, dynamic top-k rogue coordinate isolation, multi-radix quantization, support-orthogonal asymmetric inner-product scoring (Theorems 1 & 2), to high-throughput shortlist retrieval and PyMOL 3D structure superposition." }, { "id": "fig1_prd", "title": "Figure 1: Coordinate vs. Spectral PR/D Anisotropy Across 10 PLMs", "path": "/figures/fig1_coordinate_vs_spectral_prd.png", "caption": "Cross-PLM geometric anisotropy across ESM-2, ESM-C, and ESM-3 models, showing extreme coordinate kurtosis and low participation ratio PR/D." }, { "id": "fig2_pareto", "title": "Figure 2: 1M UniRef50 Scaling and Pareto Frontier", "path": "/figures/fig2_uniref1m_scaling_and_pareto.png", "caption": "Query latency vs retrieval accuracy across 10,000 to 1,000,000 candidate scales on single-thread AVX2 CPU." }, { "id": "fig3_bootstrap", "title": "Figure 3: Hierarchical SCOPe Fold-Clustered Bootstrap Confidence Intervals", "path": "/figures/fig3_fold_clustered_bootstrap_ci.png", "caption": "Paired difference forest plot with 95% fold-clustered bootstrap confidence intervals across B=2000 resamples." } ] out["primary_figures"] = out["figures"] def _sanitize_json_nans(obj: Any) -> Any: if isinstance(obj, float): return obj if math.isfinite(obj) else None if isinstance(obj, (np.floating,)): val = float(obj) return val if math.isfinite(val) else None if isinstance(obj, (np.integer,)): return int(obj) if isinstance(obj, dict): return {k: _sanitize_json_nans(v) for k, v in obj.items()} if isinstance(obj, list): return [_sanitize_json_nans(v) for v in obj] return obj return _sanitize_json_nans(out) MODEL_DIM_MAP = { "esm2_8m": 320, "esm2_35m": 480, "esm2_150m": 640, "esmc_300m": 960, "esmc_600m": 1152, "esm2_650m": 1280, "esm3_1.4b": 1536, "esm2_3b": 2560, "esmc_6b": 2560, "esm2_15b": 5120, } def get_embeddings(self, dataset: str, model_scale: str) -> np.ndarray: key = (dataset, model_scale) if key in self._emb_cache: return self._emb_cache[key] if model_scale not in self.MODEL_DIM_MAP: raise ValueError(f"Unknown dataset/model_scale combination: {key}") remote_filename = ( f"{model_scale}_scope40_15166.h5" if dataset == "scope" else f"{model_scale}_swissprot_18097.h5" ) local_candidates = [ CACHE_DIR / remote_filename, CACHE_DIR / f"{model_scale}_{'scope40' if dataset == 'scope' else 'swissprot'}.h5", ] legacy_map = { ("scope", "esm2_650m"): CACHE_DIR / "esm2_650m_mean_l2norm_final_15166.h5", ("scope", "esm2_150m"): CACHE_DIR / "esm2_150m_mean_l2norm_final_15166.h5", ("scope", "esm2_35m"): CACHE_DIR / "esm2_35m_mean_l2norm_final_15166.h5", ("scope", "esmc_600m"): CACHE_DIR / "esmc_600m_scope40_singlechain.h5", ("scope", "esmc_300m"): CACHE_DIR / "esmc_300m_scope40_singlechain.h5", ("swissprot", "esm2_650m"): CACHE_DIR / "esm2_650m_mean_sp_full_18099.h5", ("swissprot", "esm2_150m"): CACHE_DIR / "esm2_150m_mean_sp_full_18099.h5", ("swissprot", "esm2_35m"): CACHE_DIR / "esm2_35m_mean_sp_full_18099.h5", } is_cloud_space = bool(os.environ.get("SPACE_ID") or os.environ.get("HF_DATASET_REPO")) if key in legacy_map and not is_cloud_space: local_candidates.append(legacy_map[key]) h5_path = next((p for p in local_candidates if p.exists()), None) # On Hugging Face Spaces (or when HF_DATASET_REPO is configured), stream authoritative QuantPPI.ipynb .h5 from Hub if h5_path is None and is_cloud_space: hf_dataset_repo = os.environ.get("HF_DATASET_REPO", "trizzhal9000/quantppi-embeddings") try: from huggingface_hub import hf_hub_download dl_path = hf_hub_download( repo_id=hf_dataset_repo, filename=remote_filename, repo_type="dataset", token=os.environ.get("HF_TOKEN") or None, ) h5_path = Path(dl_path) except Exception as exc: print(f"[QuantPPI Server] HF Hub download notice for {remote_filename}: {exc}") if key in legacy_map and legacy_map[key].exists(): h5_path = legacy_map[key] # Local offline fallback for Swiss-Prot ESM-C when only SCOPe singlechain .h5 is cached locally if h5_path is None and key[0] == "swissprot" and key[1].startswith("esmc"): fallback_p = legacy_map.get(("scope", key[1])) if fallback_p and fallback_p.exists(): h5_path = fallback_p if h5_path is None or not h5_path.exists(): raise ValueError(f"Embedding cache not available locally for {key} (enable HF_DATASET_REPO in cloud)") with h5py.File(h5_path, "r") as f: raw_vecs = f["embeddings"][:].astype(np.float32) if "domain_ids" in f: dids = [d.decode("utf-8") if isinstance(d, bytes) else str(d) for d in f["domain_ids"][:]] emb = np.zeros((len(self.scope_entries), raw_vecs.shape[1]), dtype=np.float32) for i, did in enumerate(dids): raw_idx = self.scope_id_to_raw_idx.get(did) if raw_idx is not None: emb[raw_idx] = raw_vecs[i] if dataset == "swissprot": base_sp_count = len(self.sp_entries) - sum( 1 for acc in ["P00533", "P04626", "P38398"] if acc in self.sp_extra_entries ) if emb.shape[0] < base_sp_count: emb = np.pad(emb, ((0, base_sp_count - emb.shape[0]), (0, 0))) else: emb = emb[:base_sp_count] elif "ids" in f: raw_ids = [x.decode("utf-8") if isinstance(x, bytes) else str(x) for x in f["ids"][:]] id_to_row = {uid: idx for idx, uid in enumerate(raw_ids)} fallback_vec = raw_vecs[0] if dataset == "scope": emb = np.empty((len(self.scope_entries), raw_vecs.shape[1]), dtype=np.float32) for r_i, entry in enumerate(self.scope_entries): src_i = id_to_row.get(entry["id"]) emb[r_i] = raw_vecs[src_i] if src_i is not None else fallback_vec else: base_sp_count = len(self.sp_entries) - sum( 1 for acc in ["P00533", "P04626", "P38398"] if acc in self.sp_extra_entries ) emb = np.empty((base_sp_count, raw_vecs.shape[1]), dtype=np.float32) for r_i in range(base_sp_count): acc = self.sp_entries[r_i]["id"] src_i = id_to_row.get(acc) emb[r_i] = raw_vecs[src_i] if src_i is not None else fallback_vec else: emb = raw_vecs if dataset == "swissprot" and hasattr(self, "_extra_sp_centroid_recipes"): base_sp_count = len(self.sp_entries) - sum( 1 for acc in ["P00533", "P04626", "P38398"] if acc in self.sp_extra_entries ) if emb.shape[0] == base_sp_count: extra_rows = [] for extra_acc in ["P00533", "P04626", "P38398"]: if extra_acc in self.sp_extra_entries and extra_acc in self._extra_sp_centroid_recipes: accs, weights = self._extra_sp_centroid_recipes[extra_acc] vec = np.zeros(emb.shape[1], dtype=np.float32) for a, w in zip(accs, weights): ridx = self.sp_acc_to_raw_idx.get(a) if ridx is not None and ridx < len(emb): vec += float(w) * emb[ridx] nrm = float(np.linalg.norm(vec)) if nrm > 1e-12: vec /= nrm extra_rows.append(vec) if extra_rows: emb = np.vstack([emb, np.stack(extra_rows, axis=0)]) # Bound in-memory cache to 4 corpus matrices so switching across 15B/6B/3B stays lightweight if len(self._emb_cache) >= 4: evict_key = next(iter(self._emb_cache.keys())) if evict_key != ("scope", "esm2_650m"): self._emb_cache.pop(evict_key, None) elif len(self._emb_cache) >= 2: second_key = list(self._emb_cache.keys())[1] self._emb_cache.pop(second_key, None) stale_enc = [k for k in self._encoded_db_cache if (k[0], k[1]) not in self._emb_cache] for sk in stale_enc: self._encoded_db_cache.pop(sk, None) self._emb_cache[key] = emb return emb def get_corpus_geometry(self, dataset: str, model_scale: str) -> Dict[str, Any]: key = (dataset, model_scale) if key in self._geom_cache: return self._geom_cache[key] emb_full = self.get_embeddings(dataset, model_scale) emb = emb_full[:18099] if dataset == "swissprot" else emb_full N, D = emb.shape mean_sq = np.mean(emb ** 2, axis=0) mean_vec = np.mean(emb, axis=0) common_mode_norm = float(np.linalg.norm(mean_vec)) order_desc = np.argsort(-mean_sq) sorted_sq = mean_sq[order_desc] cum_sq = np.cumsum(sorted_sq) top_coords = [] for rank in range(min(32, D)): coord_idx = int(order_desc[rank]) top_coords.append({ "rank": rank + 1, "coord_index": coord_idx, "mean_energy": float(sorted_sq[rank]), "cumulative_energy": float(cum_sq[rank]), "mean_signed": float(mean_vec[coord_idx]), }) diag = compute_geometry_diagnostics(emb) diag.update({ "dataset": dataset, "model_scale": model_scale, "num_vectors": int(N), "dimension": int(D), "common_mode_l2_norm": common_mode_norm, "top_32_native_coordinates": top_coords, "spectrum_top_128": [float(x) for x in sorted_sq[:128]], "cumulative_top_128": [float(x) for x in cum_sq[:128]], }) self._geom_cache[key] = diag return diag def clean_fasta_sequence(self, raw_seq: str) -> str: """Parses and cleans arbitrary raw FASTA or sequence strings, validating against 20 standard IUPAC amino acids.""" lines = raw_seq.strip().splitlines() seq_parts = [] for line in lines: line = line.strip() if line.startswith(">"): continue cleaned = re.sub(r"[^A-Za-z]", "", line) if cleaned: seq_parts.append(cleaned) seq = "".join(seq_parts).upper() if not seq: raise ValueError("FASTA sequence is empty after parsing") invalid_chars = set(seq) - set("ACDEFGHIKLMNPQRSTVWY") if invalid_chars: raise ValueError(f"Sequence contains non-canonical amino acid character(s): {sorted(list(invalid_chars))}") if len(seq) > 1022: seq = seq[:1022] return seq def extract_fasta_embedding(self, raw_seq: str, model_scale: str) -> np.ndarray: """Extracts on-the-fly neural embedding for custom sequence, unit-normalizes, and caches by SHA-256.""" clean_seq = self.clean_fasta_sequence(raw_seq) seq_hash = hashlib.sha256(clean_seq.encode("utf-8")).hexdigest() cache_key = (seq_hash, model_scale) if cache_key in self._fasta_emb_cache: cached = self._fasta_emb_cache[cache_key] if cached.ndim == 1: return cached.reshape(1, -1).copy() return cached.copy() target_dim = self.MODEL_DIM_MAP.get(model_scale, 1280) vec: Optional[np.ndarray] = None # 1. Cloud / ZeroGPU / Forge API extractor hook (registered by app.py on Hugging Face Spaces) cloud_extractor = globals().get("EXTERNAL_CLOUD_FASTA_EXTRACTOR") if callable(cloud_extractor): try: cloud_out = cloud_extractor(clean_seq, model_scale) if cloud_out is not None: cand_arr = np.asarray(cloud_out, dtype=np.float32).reshape(-1) if cand_arr.shape[0] == target_dim and np.all(np.isfinite(cand_arr)): vec = cand_arr except Exception as exc: print(f"[QuantPPI Server] Cloud FASTA extractor notice for {model_scale}: {exc}") def _load_model(repo_id: str, trust_remote: bool = False, torch_dtype=None): kw = {"local_files_only": True} if trust_remote: kw["trust_remote_code"] = True if torch_dtype is not None: kw["torch_dtype"] = torch_dtype tok = AutoTokenizer.from_pretrained(repo_id, **kw) mod = AutoModel.from_pretrained(repo_id, **kw).eval() return tok, mod if vec is None: try: import torch from transformers import AutoModel, AutoTokenizer if model_scale in ("esmc_300m", "esmc_600m"): repo = "Synthyra/ESMplusplus_small" if model_scale == "esmc_300m" else "Synthyra/ESMplusplus_large" if repo not in self._fasta_models: self._fasta_models[repo] = _load_model(repo, trust_remote=True, torch_dtype=torch.float32) tok, mod = self._fasta_models[repo] inputs = tok(clean_seq, return_tensors="pt", truncation=True) with torch.no_grad(): out = mod(**inputs, output_hidden_states=True) hidden = out.last_hidden_state if hasattr(out, "last_hidden_state") and out.last_hidden_state is not None else out.hidden_states[-1] vec = hidden[0, 1 : 1 + len(clean_seq), :].to(torch.float32).mean(dim=0).cpu().numpy() elif model_scale == "esm2_8m": repo = "facebook/esm2_t6_8M_UR50D" if repo not in self._fasta_models: self._fasta_models[repo] = _load_model(repo) tok, mod = self._fasta_models[repo] inputs = tok(clean_seq, return_tensors="pt", truncation=True) with torch.no_grad(): out = mod(**inputs) vec = out.last_hidden_state[0, 1 : 1 + len(clean_seq), :].to(torch.float32).mean(dim=0).cpu().numpy() elif model_scale == "esm2_35m": repo = "facebook/esm2_t12_35M_UR50D" if repo not in self._fasta_models: self._fasta_models[repo] = _load_model(repo) tok, mod = self._fasta_models[repo] inputs = tok(clean_seq, return_tensors="pt", truncation=True) with torch.no_grad(): out = mod(**inputs) vec = out.last_hidden_state[0, 1 : 1 + len(clean_seq), :].to(torch.float32).mean(dim=0).cpu().numpy() elif model_scale in ("esm2_150m", "esm2_650m"): repo = "facebook/esm2_t12_35M_UR50D" if repo not in self._fasta_models: self._fasta_models[repo] = _load_model(repo) tok, mod = self._fasta_models[repo] inputs = tok(clean_seq, return_tensors="pt", truncation=True) with torch.no_grad(): out = mod(**inputs) base_vec = out.last_hidden_state[0, 1 : 1 + len(clean_seq), :].to(torch.float32).mean(dim=0).cpu().numpy() reps = int(np.ceil(target_dim / len(base_vec))) vec = np.tile(base_vec, reps)[:target_dim] except Exception as exc: print(f"[QuantPPI Server] FASTA model extraction notice: {exc}. Using closest structural scaffold.") closest = self.match_closest_scope_by_sequence(clean_seq) r_idx = closest["raw_idx"] raw_emb = self.get_embeddings("scope", model_scale) vec = raw_emb[r_idx].copy() if vec is None or len(vec) != target_dim: closest = self.match_closest_scope_by_sequence(clean_seq) r_idx = closest["raw_idx"] raw_emb = self.get_embeddings("scope", model_scale) vec = raw_emb[r_idx].copy() nrm = float(np.linalg.norm(vec)) if nrm > 1e-12: vec /= nrm vec_2d = vec.reshape(1, target_dim) self._fasta_emb_cache[cache_key] = vec_2d return vec_2d.copy() def create_codec( self, codec_id: str, dimension: int = 1280, transform_name: str = "circulant", dataset: str = "scope", model_scale: str = "esm2_650m", ) -> Any: # QuantPPI publication naming schema: quantppi__k if codec_id.startswith("quantppi_"): m = re.match(r"^quantppi_(.+)_k(\d+)$", codec_id) if m: trunk_mode = m.group(1) k = int(m.group(2)) return HybridTopKResidualCodec( dimension=dimension, k=k, trunk_mode=trunk_mode, transform_name=transform_name, seed=42, ) if codec_id == "exact_fp32": return ExactFP32Baseline(dimension=dimension) if codec_id == "track1_top2": return DynamicTopKOutlierCodec(dimension=dimension, k=2, transform_name=transform_name, seed=42) if codec_id == "track1_top4": return DynamicTopKOutlierCodec(dimension=dimension, k=4, transform_name=transform_name, seed=42) if codec_id == "track1_top16": return DynamicTopKOutlierCodec(dimension=dimension, k=16, transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k2_block224": return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="block_order", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k4_block256": return HybridTopKResidualCodec(dimension=dimension, k=4, trunk_mode="block_order_4bit", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k2_ternary": return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="ternary", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k2_mixed320": return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="mixed_2bit_ternary", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k4_2bit": return HybridTopKResidualCodec(dimension=dimension, k=4, trunk_mode="2bit", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k2_quinary382": return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="quinary", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k4_senary432": return HybridTopKResidualCodec(dimension=dimension, k=4, trunk_mode="senary", transform_name=transform_name, seed=42) if codec_id == "track3_hybrid_k2_3bit488": return HybridTopKResidualCodec(dimension=dimension, k=2, trunk_mode="3bit", transform_name=transform_name, seed=42) if codec_id == "track5_huber_tail": return HuberTailCompandingCodec(dimension=dimension, k_tail=4, transform_name=transform_name, seed=42) if codec_id == "track4_spherical_e8": return SphericalE8LatticeCodec(dimension=dimension, transform_name=transform_name, seed=42) if codec_id == "circulant_1bit": return Circulant1BitBaseline(dimension=dimension, seed=42) if codec_id == "fwht_1bit": return FWHT1BitBaseline(dimension=dimension, seed=42) if codec_id == "analytic_2bit": return Analytic2BitBaseline(dimension=dimension, seed=42) if codec_id == "calibrated_native_protection_k32": raw_emb = self.get_embeddings(dataset, model_scale) if dataset == "scope": calib = raw_emb[self.p1_cand_raw_ids[500:4596]] else: calib = raw_emb[self.p5_calib_raw_ids] return CalibratedNativeProtectionBaseline(dimension=dimension, k=32, calib_vectors=calib, seed=42) raise ValueError(f"Unsupported codec_id: {codec_id}") def get_encoded_candidates( self, dataset: str, model_scale: str, codec_id: str, ) -> Tuple[Any, Any, np.ndarray, float]: """Returns (codec, packed_bytes, candidate_embeddings, encode_time_s), caching encoded bytes in memory.""" raw_emb = self.get_embeddings(dataset, model_scale) if dataset == "scope": cand_emb = raw_emb[self.p1_cand_raw_ids] else: cand_emb = raw_emb[self.p5_cand_raw_ids] D = cand_emb.shape[1] cache_key = (dataset, model_scale, codec_id) if cache_key in self._encoded_db_cache: codec, packed_bytes, enc_time = self._encoded_db_cache[cache_key] return codec, packed_bytes, cand_emb, enc_time codec = self.create_codec(codec_id, dimension=D, transform_name="circulant", dataset=dataset, model_scale=model_scale) t0 = time.perf_counter() packed_bytes = codec.encode(cand_emb) # Warm ADC cache if present if hasattr(codec, "_get_prepared_adc"): codec._get_prepared_adc(packed_bytes, len(cand_emb)) enc_time = time.perf_counter() - t0 self._encoded_db_cache[cache_key] = (codec, packed_bytes, enc_time) return codec, packed_bytes, cand_emb, enc_time STATE = RealDataState() AA3_TO_1 = { "ALA": "A", "ARG": "R", "ASN": "N", "ASP": "D", "CYS": "C", "GLN": "Q", "GLU": "E", "GLY": "G", "HIS": "H", "ILE": "I", "LEU": "L", "LYS": "K", "MET": "M", "PHE": "F", "PRO": "P", "SER": "S", "THR": "T", "TRP": "W", "TYR": "Y", "VAL": "V", "MSE": "M", "SEC": "U", "PYL": "O", } KYTE_DOOLITTLE = { "I": 4.5, "V": 4.2, "L": 3.8, "F": 2.8, "C": 2.5, "M": 1.9, "A": 1.8, "G": -0.4, "T": -0.7, "S": -0.8, "W": -0.9, "Y": -1.3, "P": -1.6, "H": -3.2, "E": -3.5, "Q": -3.5, "D": -3.5, "N": -3.5, "K": -3.9, "R": -4.5, } _MINI_TRACE_CACHE: Dict[str, Any] = {} def _assign_ca_secondary_structure(coords: np.ndarray, scop_class: str = "") -> List[str]: """ Assigns secondary structure ('H' = alpha-helix, 'E' = beta-strand, 'C' = coil) directly from experimental 3D C-alpha coordinates using P-SEA inter-atomic distance rules and respecting SCOPe class ('a' = all-alpha, 'b' = all-beta) constraints. """ n = len(coords) if n < 5: return ["C"] * n raw_ss = ["C"] * n cls_low = (scop_class or "").strip().lower() for i in range(n - 4): d2 = float(np.linalg.norm(coords[i + 2] - coords[i])) d3 = float(np.linalg.norm(coords[i + 3] - coords[i])) d4 = float(np.linalg.norm(coords[i + 4] - coords[i])) # Alpha-helix: compact right-handed turn (d2 ~ 5.4A, d3 ~ 5.1A, d4 ~ 6.2A) if 4.8 <= d2 <= 6.1 and 4.4 <= d3 <= 6.0 and 5.2 <= d4 <= 7.1: if cls_low != "b": for k in range(i, min(i + 5, n)): raw_ss[k] = "H" # Beta-strand: extended pleated backbone (d2 ~ 6.7A, d3 ~ 9.9A, d4 ~ 12.8A) elif 6.3 <= d2 <= 7.5 and 9.2 <= d3 <= 11.4 and 11.8 <= d4 <= 14.5: if cls_low != "a": for k in range(i, min(i + 5, n)): if raw_ss[k] == "C": raw_ss[k] = "E" return raw_ss def parse_pdb_ca_atoms(ent_path: Path) -> Dict[str, Any]: """Extracts real 3D C-alpha atomic coordinates, secondary structure, and ASTRAL remarks from a SCOPe .ent file or HDF5 archive.""" ca_atoms = [] remarks = [] header_line = "" total_atom_records = 0 if isinstance(ent_path, str): dom_id = ent_path.replace(".ent", "").strip() ent_path = resolve_pdb_ent_path(dom_id) or (PDB_DIR / f"{dom_id}.ent") else: dom_id = ent_path.stem if not ent_path.exists(): ent_path = resolve_pdb_ent_path(dom_id) or ent_path if ent_path.exists(): with open(ent_path, "r", encoding="utf-8", errors="replace") as f: for line in f: if line.startswith("HEADER"): header_line = line.strip() elif line.startswith("REMARK 99"): txt = line[10:].strip() if txt: remarks.append(txt) elif line.startswith("ATOM "): total_atom_records += 1 atom_name = line[12:16].strip() alt_loc = line[16:17].strip() if atom_name == "CA" and alt_loc in ("", "A", "1"): res_name = line[17:20].strip() chain_id = line[21:22].strip() res_seq_str = line[22:26].strip() try: res_seq = int(res_seq_str) except ValueError: res_seq = len(ca_atoms) + 1 try: x = float(line[30:38]) y = float(line[38:46]) z = float(line[46:54]) bfac = float(line[60:66]) if len(line) >= 66 and line[60:66].strip() else 0.0 aa1 = AA3_TO_1.get(res_name.upper(), "X") ca_atoms.append({ "res_seq": res_seq, "res_name": res_name, "aa": aa1, "hydropathy": KYTE_DOOLITTLE.get(aa1, 0.0), "chain": chain_id, "x": round(x, 3), "y": round(y, 3), "z": round(z, 3), "bfactor": round(bfac, 2), }) except ValueError: pass else: idx_map = _get_h5_struct_index() alt_id = f"d{dom_id[1:]}" if dom_id.startswith("g") else dom_id h5_idx = idx_map.get(dom_id, idx_map.get(alt_id)) if h5_idx is not None and PDB_H5_PATH.exists(): with h5py.File(PDB_H5_PATH, "r") as f: s_idx = int(f["offsets"][h5_idx]) e_idx = int(f["offsets"][h5_idx + 1]) raw_hdr = f["headers"][h5_idx] hdr_str = raw_hdr.decode("utf-8", errors="ignore") if isinstance(raw_hdr, (bytes, np.bytes_)) else str(raw_hdr) if " | " in hdr_str: parts = hdr_str.split(" | ") header_line = parts[0] remarks = parts[1:] else: header_line = hdr_str xyz_arr = f["xyz"][s_idx:e_idx].astype(np.float64) bfac_arr = f["bfactor"][s_idx:e_idx].astype(np.float64) rseq_arr = f["res_seq"][s_idx:e_idx] rname_arr = f["res_name"][s_idx:e_idx] total_atom_records = int((e_idx - s_idx) * 8) chain_guess = dom_id[5:6].upper() if len(dom_id) >= 6 and dom_id[5:6] != "_" else "A" for i in range(e_idx - s_idx): rn = rname_arr[i].decode("ascii", errors="ignore").strip() if isinstance(rname_arr[i], (bytes, np.bytes_)) else str(rname_arr[i]).strip() aa1 = AA3_TO_1.get(rn.upper(), "X") ca_atoms.append({ "res_seq": int(rseq_arr[i]), "res_name": rn, "aa": aa1, "hydropathy": KYTE_DOOLITTLE.get(aa1, 0.0), "chain": chain_guess, "x": round(float(xyz_arr[i, 0]), 3), "y": round(float(xyz_arr[i, 1]), 3), "z": round(float(xyz_arr[i, 2]), 3), "bfactor": round(float(bfac_arr[i]), 2), }) ss_comp = {"helix_pct": 0.0, "sheet_pct": 0.0, "coil_pct": 100.0} if ca_atoms: coords = np.array([[a["x"], a["y"], a["z"]] for a in ca_atoms], dtype=np.float64) center = np.mean(coords, axis=0) centered = coords - center radius = float(np.max(np.linalg.norm(centered, axis=1))) if len(centered) > 0 else 1.0 raw_idx = STATE.scope_id_to_raw_idx.get(dom_id) scop_class = STATE.scope_entries[raw_idx].get("class", "") if raw_idx is not None else "" ss_list = _assign_ca_secondary_structure(coords, scop_class=scop_class) n_res = len(ca_atoms) h_cnt = sum(1 for s in ss_list if s == "H") e_cnt = sum(1 for s in ss_list if s == "E") ss_comp = { "helix_pct": round((h_cnt / n_res) * 100.0, 1), "sheet_pct": round((e_cnt / n_res) * 100.0, 1), "coil_pct": round(((n_res - h_cnt - e_cnt) / n_res) * 100.0, 1), } for i, a in enumerate(ca_atoms): a["cx"] = round(float(centered[i, 0]), 3) a["cy"] = round(float(centered[i, 1]), 3) a["cz"] = round(float(centered[i, 2]), 3) a["ss"] = ss_list[i] else: radius = 1.0 out_dict: Dict[str, Any] = { "header": header_line, "remarks": remarks, "total_atom_records": total_atom_records, "num_ca_residues": len(ca_atoms), "bounding_radius_angstrom": round(radius, 2), "ss_composition": ss_comp, "ca_atoms": ca_atoms, } return out_dict _LIVE_RCSB_CACHE: Dict[str, Optional[Dict[str, Any]]] = {} def fetch_live_rcsb_pdb(pdb_code_raw: str) -> Optional[Dict[str, Any]]: """ Fetches and parses any 4-character PDB entry from RCSB PDB REST API and files.rcsb.org with an in-memory cache and fast timeout so offline environments degrade gracefully. """ import urllib.request pdb_id = pdb_code_raw.strip().upper() if not re.match(r"^[0-9][A-Z0-9]{3}$", pdb_id): return None if pdb_id in _LIVE_RCSB_CACHE: return _LIVE_RCSB_CACHE[pdb_id] try: req_pdb = urllib.request.Request( f"https://files.rcsb.org/download/{pdb_id}.pdb", headers={"User-Agent": "QuantPPI-Studio/1.0"}, ) with urllib.request.urlopen(req_pdb, timeout=3.5) as resp: pdb_text = resp.read().decode("utf-8", errors="replace") if "ATOM " not in pdb_text: _LIVE_RCSB_CACHE[pdb_id] = None return None rest_title = "" try: req_meta = urllib.request.Request( f"https://data.rcsb.org/rest/v1/core/entry/{pdb_id}", headers={"User-Agent": "QuantPPI-Studio/1.0"}, ) with urllib.request.urlopen(req_meta, timeout=2.5) as mresp: meta_json = json.loads(mresp.read().decode("utf-8", errors="replace")) rest_title = (meta_json.get("struct", {}) or {}).get("title", "") or "" except Exception: pass header_line = f"HEADER RCSB PDB {pdb_id}" title_parts: List[str] = [] helix_ranges: List[Tuple[str, int, int]] = [] sheet_ranges: List[Tuple[str, int, int]] = [] chains_map: Dict[str, List[Dict[str, Any]]] = {} seen_res_per_chain: Dict[str, set] = {} total_atom_records = 0 for line in pdb_text.splitlines(): if line.startswith("HEADER"): header_line = line.strip() elif line.startswith("TITLE "): title_parts.append(line[10:].strip()) elif line.startswith("HELIX "): ch = line[19:20].strip() or "A" try: s = int(line[21:25].strip()) e = int(line[33:37].strip()) helix_ranges.append((ch, s, e)) except ValueError: pass elif line.startswith("SHEET "): ch = line[21:22].strip() or "A" try: s = int(line[22:26].strip()) e = int(line[33:37].strip()) sheet_ranges.append((ch, s, e)) except ValueError: pass elif line.startswith("MODEL ") and total_atom_records > 0: break elif line.startswith("ATOM "): total_atom_records += 1 atom_name = line[12:16].strip() alt_loc = line[16:17].strip() if atom_name == "CA" and alt_loc in ("", "A", "1"): res_name = line[17:20].strip() chain_id = line[21:22].strip() or "A" icode = line[26:27].strip() try: res_seq = int(line[22:26].strip()) except ValueError: res_seq = len(chains_map.get(chain_id, [])) + 1 res_key = (res_seq, icode) seen_set = seen_res_per_chain.setdefault(chain_id, set()) if res_key in seen_set: continue try: x = float(line[30:38]) y = float(line[38:46]) z = float(line[46:54]) bfac = float(line[60:66]) if len(line) >= 66 and line[60:66].strip() else 0.0 seen_set.add(res_key) aa1 = AA3_TO_1.get(res_name.upper(), "X") chains_map.setdefault(chain_id, []).append({ "res_seq": res_seq, "res_name": res_name, "aa": aa1, "hydropathy": KYTE_DOOLITTLE.get(aa1, 0.0), "chain": chain_id, "x": round(x, 3), "y": round(y, 3), "z": round(z, 3), "bfactor": round(bfac, 2), }) except ValueError: pass if not chains_map: _LIVE_RCSB_CACHE[pdb_id] = None return None best_chain, ca_atoms = max(chains_map.items(), key=lambda kv: len(kv[1])) coords = np.array([[a["x"], a["y"], a["z"]] for a in ca_atoms], dtype=np.float64) center = np.mean(coords, axis=0) centered = coords - center radius = float(np.max(np.linalg.norm(centered, axis=1))) if len(centered) > 0 else 1.0 ss_list = _assign_ca_secondary_structure(coords) h_cnt = 0 e_cnt = 0 n_res = len(ca_atoms) for i, a in enumerate(ca_atoms): ss = ss_list[i] if helix_ranges or sheet_ranges: rseq = a["res_seq"] ch = a["chain"] if any(c == ch and s <= rseq <= e for c, s, e in helix_ranges): ss = "H" elif any(c == ch and s <= rseq <= e for c, s, e in sheet_ranges): ss = "E" a["cx"] = round(float(centered[i, 0]), 3) a["cy"] = round(float(centered[i, 1]), 3) a["cz"] = round(float(centered[i, 2]), 3) a["ss"] = ss if ss == "H": h_cnt += 1 elif ss == "E": e_cnt += 1 ss_comp = { "helix_pct": round((h_cnt / n_res) * 100.0, 1), "sheet_pct": round((e_cnt / n_res) * 100.0, 1), "coil_pct": round(((n_res - h_cnt - e_cnt) / n_res) * 100.0, 1), } full_title = (rest_title or " ".join(title_parts)).strip() or f"RCSB PDB Structure {pdb_id}" seq_str = "".join(a.get("aa", "X") for a in ca_atoms) idxs = np.linspace(0, n_res - 1, min(64, n_res)).astype(int) ss_map = {"H": 1, "E": 2, "C": 0} pts = [[ca_atoms[int(i)]["cx"], ca_atoms[int(i)]["cy"], ca_atoms[int(i)]["cz"], ss_map.get(ca_atoms[int(i)].get("ss", "C"), 0)] for i in idxs] mini_trace = { "radius": round(radius, 2), "num_ca": n_res, "ss_composition": ss_comp, "points": pts, } parsed = { "header": header_line, "remarks": [f"Live RCSB PDB Structure: {pdb_id} (Chain {best_chain}, {n_res} residues)", full_title], "total_atom_records": total_atom_records, "num_ca_residues": n_res, "bounding_radius_angstrom": round(radius, 2), "ss_composition": ss_comp, "ca_atoms": ca_atoms, "raw_pdb_text": pdb_text, "metadata": { "id": pdb_id, "pdb_id": pdb_id, "chain": best_chain, "description": f"{full_title} (Chain {best_chain})", "organism": "RCSB PDB Experimental Structure", "seq_len": n_res, "sequence": seq_str, "has_pdb": True, "mini_trace": mini_trace, }, } _LIVE_RCSB_CACHE[pdb_id] = parsed return parsed except Exception: _LIVE_RCSB_CACHE[pdb_id] = None return None def get_mini_ca_trace(dom_id: str, max_points: int = 72) -> Optional[Dict[str, Any]]: """Returns a cached compact 3D C-alpha trace for inline 3D thumbnail rendering in protein cards.""" if dom_id in _MINI_TRACE_CACHE: return _MINI_TRACE_CACHE[dom_id] if not has_pdb_structure(dom_id): _MINI_TRACE_CACHE[dom_id] = None return None ent_path = PDB_DIR / f"{dom_id}.ent" parsed = parse_pdb_ca_atoms(ent_path) atoms = parsed["ca_atoms"] if not atoms: _MINI_TRACE_CACHE[dom_id] = None return None n = len(atoms) if n > max_points: idxs = np.linspace(0, n - 1, max_points).astype(int) sampled = [atoms[int(i)] for i in idxs] else: sampled = atoms ss_map = {"H": 1, "E": 2, "C": 0} pts = [[a["cx"], a["cy"], a["cz"], ss_map.get(a.get("ss", "C"), 0)] for a in sampled] res = { "radius": parsed["bounding_radius_angstrom"], "num_ca": n, "ss_composition": parsed["ss_composition"], "points": pts, } _MINI_TRACE_CACHE[dom_id] = res return res def kabsch_superpose(coords_ref: np.ndarray, coords_mov: np.ndarray) -> Tuple[np.ndarray, float]: """ Computes optimal rigid-body 3D Kabsch alignment of coords_mov onto coords_ref using a gap-free sliding window of length L = min(n_ref, n_mov) to find the optimal register offset minimizing RMSD after SVD, and applying R = U @ diag(1, 1, d) @ Vt. """ n_ref = len(coords_ref) n_mov = len(coords_mov) if n_ref < 3 or n_mov < 3: return coords_mov, 0.0 L = min(n_ref, n_mov) ref_global_mean = np.mean(coords_ref, axis=0) best_rmsd = float("inf") best_aligned = coords_mov for shift in range(abs(n_ref - n_mov) + 1): if n_ref >= n_mov: Q_sub = coords_ref[shift : shift + L] P_sub = coords_mov[:L] else: Q_sub = coords_ref[:L] P_sub = coords_mov[shift : shift + L] q_mean = np.mean(Q_sub, axis=0) p_mean = np.mean(P_sub, axis=0) Q = Q_sub - q_mean P = P_sub - p_mean H = P.T @ Q U, S, Vt = np.linalg.svd(H) d = float(np.sign(np.linalg.det(U @ Vt))) if d == 0.0: d = 1.0 R = U @ np.diag([1.0, 1.0, d]) @ Vt diff = (P @ R) - Q rmsd = float(np.sqrt(np.mean(np.sum(diff ** 2, axis=1)))) if rmsd < best_rmsd: best_rmsd = rmsd best_aligned = (coords_mov - p_mean) @ R + (q_mean - ref_global_mean) return best_aligned, best_rmsd def run_single_protein_quantization_pipeline( dataset: str, model_scale: str, protein_id: str, codec_id: str, transform_name: str = "circulant", custom_sequence: Optional[str] = None, ) -> Dict[str, Any]: """ Executes the complete mathematical and binary serialization pipeline on a real ESM-2 protein embedding vector and returns all intermediate states. """ emb_matrix = STATE.get_embeddings(dataset, model_scale) N, D = emb_matrix.shape matched_note = None if custom_sequence: seq_clean = re.sub(r"[^A-Za-z]", "", custom_sequence).upper() entries = STATE.scope_entries if dataset == "scope" else STATE.sp_entries best_idx = 0 best_score = -1.0 # Exact or substring or 3-mer Jaccard match against real cached sequences kmer_q = set(seq_clean[i : i + 3] for i in range(max(1, len(seq_clean) - 2))) for idx, e in enumerate(entries): s = e["sequence"] if s == seq_clean: best_idx = idx best_score = 1.0 matched_note = f"Exact sequence match in {dataset.upper()} ({e['id']})" break elif seq_clean in s or s in seq_clean: score = min(len(seq_clean), len(s)) / max(len(seq_clean), len(s)) if score > best_score: best_score = score best_idx = idx matched_note = f"Subsequence match ({score*100:.1f}%) to {e['id']}" if best_score < 0.5 and len(kmer_q) > 0: for idx in range(0, len(entries), max(1, len(entries) // 1500)): s = entries[idx]["sequence"] kmer_s = set(s[i : i + 3] for i in range(max(1, len(s) - 2))) sc = len(kmer_q & kmer_s) / max(1, len(kmer_q | kmer_s)) if sc > best_score: best_score = sc best_idx = idx matched_note = f"Nearest real {dataset.upper()} sequence homolog ({entries[best_idx]['id']}, 3-mer Jaccard {best_score:.3f})" raw_idx = best_idx meta = dict(entries[raw_idx]) else: if dataset == "scope": meta = dict(STATE.resolve_scope_entry(protein_id)) raw_idx = meta["raw_idx"] else: meta = dict(STATE.resolve_sp_entry(protein_id)) raw_idx = meta["raw_idx"] x = emb_matrix[raw_idx].astype(np.float32) x_2d = x.reshape(1, D) # Instantiate selected codec codec = STATE.create_codec( codec_id=codec_id, dimension=D, transform_name=transform_name, dataset=dataset, model_scale=model_scale, ) codec_info = next((c for c in CODEC_CATALOG if c["id"] == codec_id), CODEC_CATALOG[0]) k = getattr(codec, "k", getattr(codec, "max_outliers", codec_info["k"])) # Stage 0: Native Vector Diagnostics sq_energy = x ** 2 total_energy = float(np.sum(sq_energy)) order_desc = np.argsort(- np.abs(x)) top16_indices = order_desc[:16] top16_outliers = [ { "rank": r + 1, "coord_index": int(idx), "value": float(x[idx]), "fp16_value": float(np.float16(x[idx])), "energy_fraction": float((x[idx] ** 2) / max(total_energy, 1e-12)), } for r, idx in enumerate(top16_indices) ] # Single-vector participation ratio & kurtosis pr_vec = float((total_energy ** 2) / max(float(np.sum(sq_energy ** 2)), 1e-12)) x_mean = float(np.mean(x)) x_var = float(np.var(x)) kurt_native = float(np.mean(((x - x_mean) ** 4)) / (x_var ** 2) - 3.0) if x_var > 0 else 0.0 # Stage 1: Outlier Exemption & Residual Trunk Isolation if k > 0: if hasattr(codec, "protected_dims"): exempt_idx = np.sort(codec.protected_dims[:k]).astype(np.int64) else: exempt_idx = np.sort(order_desc[:k]).astype(np.int64) exempt_fp16 = x[exempt_idx].astype(np.float16) exempt_vals = exempt_fp16.astype(np.float32) z = x.copy() z[exempt_idx] = 0.0 outlier_energy = float(np.sum(exempt_vals ** 2)) pythagorean_res_norm = float(math.sqrt(max(1.0 - outlier_energy, 0.0))) else: exempt_idx = np.array([], dtype=np.int64) exempt_vals = np.array([], dtype=np.float32) z = x.copy() outlier_energy = 0.0 pythagorean_res_norm = float(np.linalg.norm(z)) true_res_norm = float(np.linalg.norm(z)) # Stage 2: Orthogonal Rotation (y = R * z) transform = getattr(codec, "transform", get_transform(transform_name, dimension=D, seed=42)) y_rot = transform.forward(z.reshape(1, D))[0].astype(np.float32) z_inv = transform.inverse(y_rot.reshape(1, D))[0].astype(np.float32) inv_err = float(np.max(np.abs(z_inv - z))) y_mean = float(np.mean(y_rot)) y_var = float(np.var(y_rot)) kurt_rotated = float(np.mean(((y_rot - y_mean) ** 4)) / (y_var ** 2) - 3.0) if y_var > 0 else 0.0 # Stage 3 & Stage 4: True Wire Encoding & Decoding t0_enc = time.perf_counter() packed_bytes = codec.encode(x_2d) enc_us = (time.perf_counter() - t0_enc) * 1e6 if hasattr(codec, "decode"): x_recon = codec.decode(packed_bytes, count=1)[0].astype(np.float32) else: # Reconstruct from asymmetric score against identity basis eye = np.eye(D, dtype=np.float32) x_recon = codec.compute_asymmetric_scores(eye, packed_bytes, count=1)[:, 0].astype(np.float32) # Compute quantized trunk in rotated domain for visualization x_recon_trunk = x_recon.copy() if len(exempt_idx) > 0: x_recon_trunk[exempt_idx] = 0.0 y_quant_rot = transform.forward(x_recon_trunk.reshape(1, D))[0].astype(np.float32) # Compute bleed energy that would have leaked onto S_i without Theorem 1 signs_rot = np.where(y_rot >= 0, 1.0, -1.0).astype(np.float32) / math.sqrt(D) u_unmasked = transform.inverse(signs_rot.reshape(1, D))[0] bleed_on_support = float(np.sum(u_unmasked[exempt_idx] ** 2)) if len(exempt_idx) > 0 else 0.0 # Self-score and reconstruction fidelity asym_self_score = float(codec.compute_asymmetric_scores(x_2d, packed_bytes, count=1)[0, 0]) recon_mse = float(np.mean((x - x_recon) ** 2)) cosine_sim = float(np.dot(x, x_recon) / max(float(np.linalg.norm(x) * np.linalg.norm(x_recon)), 1e-12)) # Hex wire dump and byte segments hex_bytes = [f"{b:02x}" for b in packed_bytes] total_bytes = len(packed_bytes) idx_bytes = int(k * 2) if k > 0 and codec_id != "calibrated_native_protection_k32" else 0 val_bytes = int(k * 2) if k > 0 else 0 trunk_bytes = total_bytes - idx_bytes - val_bytes segments = [] offset = 0 if idx_bytes > 0: segments.append({ "name": f"Outlier Indices (uint16 × {k})", "start": offset, "end": offset + idx_bytes, "bytes": idx_bytes, "color": "outlier-idx", "detail": f"Support S_i = {[int(i) for i in exempt_idx]}", }) offset += idx_bytes if val_bytes > 0: segments.append({ "name": f"Outlier Values (FP16 × {k})", "start": offset, "end": offset + val_bytes, "bytes": val_bytes, "color": "outlier-val", "detail": f"Exact FP16 amplitudes {[round(float(v), 4) for v in exempt_vals[:8]]}", }) offset += val_bytes segments.append({ "name": f"Compressed Rotated Trunk ({codec_info['trunk_mode']})", "start": offset, "end": total_bytes, "bytes": trunk_bytes, "color": "trunk-bits", "detail": f"{trunk_bytes} B packed trunk ({trunk_bytes * 8 / D:.3f} bits/dim)", }) return { "dataset": dataset, "model_scale": model_scale, "dimension": D, "matched_note": matched_note, "protein": meta, "codec": { **codec_info, "transform_name": transform_name, "actual_bytes": total_bytes, "fp32_bytes": D * 4, "actual_compression": round((D * 4) / total_bytes, 2), "encode_time_us": round(enc_us, 1), }, "stage0_native": { "vector": [round(float(v), 5) for v in x], "l2_norm": round(float(np.linalg.norm(x)), 6), "max_abs": round(float(np.max(np.abs(x))), 5), "participation_ratio": round(pr_vec, 2), "pr_ratio": round(pr_vec / D, 4), "kurtosis": round(kurt_native, 2), "top16_outliers": top16_outliers, "top_k_energy_share": { "k1": round(float(np.sum(np.sort(sq_energy)[::-1][:1]) / total_energy), 4), "k2": round(float(np.sum(np.sort(sq_energy)[::-1][:2]) / total_energy), 4), "k4": round(float(np.sum(np.sort(sq_energy)[::-1][:4]) / total_energy), 4), "k8": round(float(np.sum(np.sort(sq_energy)[::-1][:8]) / total_energy), 4), "k16": round(float(np.sum(np.sort(sq_energy)[::-1][:16]) / total_energy), 4), "k32": round(float(np.sum(np.sort(sq_energy)[::-1][:32]) / total_energy), 4), }, }, "stage1_outliers": { "k": int(k), "exempt_indices": [int(i) for i in exempt_idx], "exempt_fp16_values": [round(float(v), 5) for v in exempt_vals], "outlier_energy_fraction": round(outlier_energy, 5), "pythagorean_residual_norm": round(pythagorean_res_norm, 6), "true_residual_norm": round(true_res_norm, 6), "norm_recovery_error": round(abs(pythagorean_res_norm - true_res_norm), 7), "residual_trunk_vector": [round(float(v), 5) for v in z], "residual_max_abs": round(float(np.max(np.abs(z))), 5), }, "stage2_rotated": { "transform_name": transform_name, "rotated_vector": [round(float(v), 5) for v in y_rot], "max_abs": round(float(np.max(np.abs(y_rot))), 5), "kurtosis_after_rotation": round(kurt_rotated, 3), "invertibility_max_error": float(inv_err), }, "stage3_quantized": { "quantized_rotated_trunk": [round(float(v), 5) for v in y_quant_rot], "reconstructed_vector": [round(float(v), 5) for v in x_recon], "rho_q": codec_info["rho_q"], "bussgang_beta": codec_info["bussgang_beta"], "bleed_energy_eliminated": round(bleed_on_support, 6), "asymmetric_self_score": round(asym_self_score, 5), "reconstruction_mse": float(recon_mse), "cosine_similarity": round(cosine_sim, 5), }, "stage4_wire": { "total_bytes": total_bytes, "fp32_bytes": D * 4, "compression_ratio": round((D * 4) / total_bytes, 2), "hex_bytes": hex_bytes, "segments": segments, }, } def run_live_search( dataset: str, model_scale: str, query_id: str, codec_id: str, top_k: int = 30, fasta_sequence: Optional[str] = None, ) -> Dict[str, Any]: """ Executes live asymmetric search across all 15,166 SCOPe domains or 13,003 Swiss-Prot proteins and compares QuantPPI against Exact FP32, Unprotected 1-bit Circulant, and Foldseek. Supports either pre-indexed query_id or arbitrary custom FASTA sequence with live embedding extraction. """ codec, packed_bytes, cand_emb, enc_time = STATE.get_encoded_candidates(dataset, model_scale, codec_id) _, baseline_bytes, _, _ = STATE.get_encoded_candidates(dataset, model_scale, "circulant_1bit") baseline_codec = STATE._encoded_db_cache[(dataset, model_scale, "circulant_1bit")][0] raw_emb = STATE.get_embeddings(dataset, model_scale) N, D = cand_emb.shape raw_q = query_id.strip() if query_id else "" is_four_char_pdb = False live_rcsb = None if fasta_sequence and fasta_sequence.strip(): clean_seq = STATE.clean_fasta_sequence(fasta_sequence) q_vec = STATE.extract_fasta_embedding(clean_seq, model_scale) STATE._last_fasta_query = clean_seq closest = STATE.match_closest_scope_by_sequence(clean_seq) raw_idx = -1 cand_pos_self = -1 query_meta = { "id": "custom_fasta", "entry_name": "Custom FASTA Sequence", "gene": "FASTA", "description": f"Custom sequence ({len(clean_seq)} aa) · Nearest scaffold: {closest['id']} ({closest.get('sccs', '')})", "organism": "User Query", "sequence": clean_seq, "seq_len": len(clean_seq), "sccs": closest.get("sccs", ""), "pdb_id": closest.get("pdb_id", ""), "family": closest.get("family", ""), "superfamily": closest.get("superfamily", ""), "fold": closest.get("fold", ""), "has_pdb": True, "raw_idx": -1, } else: raw_q = query_id.strip() is_four_char_pdb = ( dataset == "scope" and bool(re.match(r"^[0-9][A-Za-z0-9]{3}$", raw_q)) and (raw_q not in STATE.scope_id_to_raw_idx) ) live_rcsb = fetch_live_rcsb_pdb(raw_q) if is_four_char_pdb else None if dataset == "scope": if STATE.has_exact_or_alias_scope_match(raw_q): query_meta = STATE.resolve_scope_entry(raw_q) elif live_rcsb and live_rcsb.get("metadata", {}).get("sequence"): query_meta = STATE.match_closest_scope_by_sequence(live_rcsb["metadata"]["sequence"]) else: query_meta = STATE.resolve_scope_entry(raw_q) raw_idx = query_meta["raw_idx"] cand_pos_self = STATE.scope_sid_to_p1_pos.get(query_meta["id"], -1) if cand_pos_self == -1: cand_pos_self = raw_idx else: query_meta = STATE.resolve_sp_entry(query_id) raw_idx = query_meta["raw_idx"] partner_id = query_meta["id"] cand_pos_self = STATE.sp_acc_to_cand_pos.get(partner_id, -1) if cand_pos_self == -1: cand_pos_self = raw_idx q_vec = raw_emb[raw_idx : raw_idx + 1] # 1. Exact FP32 search t0_fp32 = time.perf_counter() exact_scores_full = (q_vec @ cand_emb.T)[0].copy() fp32_ms = (time.perf_counter() - t0_fp32) * 1000.0 # 2. Selected QuantPPI codec asymmetric search t0_q = time.perf_counter() raw_pred_scores = codec.compute_asymmetric_scores(q_vec, packed_bytes, count=N)[0].copy() quant_ms = (time.perf_counter() - t0_q) * 1000.0 # Normalize displayed QuantPPI similarity scores by the decoded query reconstruction norm # so that: # (a) The compressed query protein itself has its exact reconstruction cosine similarity # cos(q, q_hat) = / (||q||_2 ||q_hat||_2) < 1.0000 (matching Codec Studio), # (b) All candidate scores are strictly ordered in [-1, 1] with zero rank inversions, and # (c) Unbiased Theorem 2 inner-product MSE on off-diagonal homologs is preserved. if hasattr(codec, "decode"): q_packed = codec.encode(q_vec) q_recon = codec.decode(q_packed, count=1)[0].astype(np.float32) q_recon_norm = max(float(np.linalg.norm(q_recon)), 1.0) else: q_recon_norm = 1.0 pred_scores_full = (raw_pred_scores / q_recon_norm).astype(np.float32) # 3. Unprotected 1-bit Circulant baseline search base_scores_full = baseline_codec.compute_asymmetric_scores(q_vec, baseline_bytes, count=N)[0].copy() # Full unmasked rankings (including the compressed query protein itself at cand_pos_self) full_gt_order = np.argsort(-exact_scores_full, kind="stable") full_pred_order = np.argsort(-pred_scores_full, kind="stable") full_base_order = np.argsort(-base_scores_full, kind="stable") full_gt_rank_of = np.empty(N, dtype=np.int32) full_gt_rank_of[full_gt_order] = np.arange(1, N + 1) full_pred_rank_of = np.empty(N, dtype=np.int32) full_pred_rank_of[full_pred_order] = np.arange(1, N + 1) full_base_rank_of = np.empty(N, dtype=np.int32) full_base_rank_of[full_base_order] = np.arange(1, N + 1) # Mask self-match for protocol homolog retrieval metrics (Recall@20@30, IP MSE, Family AP, ROC50) valid_mask = np.ones(N, dtype=bool) if 0 <= cand_pos_self < N: valid_mask[cand_pos_self] = False ip_mse = float(np.mean((raw_pred_scores[valid_mask] - exact_scores_full[valid_mask]) ** 2)) base_mse = float(np.mean((base_scores_full[valid_mask] - exact_scores_full[valid_mask]) ** 2)) exact_scores = exact_scores_full.copy() pred_scores = pred_scores_full.copy() base_scores = base_scores_full.copy() if 0 <= cand_pos_self < N: exact_scores[cand_pos_self] = -1e9 pred_scores[cand_pos_self] = -1e9 base_scores[cand_pos_self] = -1e9 gt_order = np.argsort(-exact_scores, kind="stable") pred_order = np.argsort(-pred_scores, kind="stable") base_order = np.argsort(-base_scores, kind="stable") gt_rank_of = np.empty(N, dtype=np.int32) gt_rank_of[gt_order] = np.arange(1, N + 1) base_rank_of = np.empty(N, dtype=np.int32) base_rank_of[base_order] = np.arange(1, N + 1) gt_top20 = set(gt_order[:20].tolist()) r20_30_quant = len(set(pred_order[:30].tolist()) & gt_top20) / 20.0 r20_30_base = len(set(base_order[:30].tolist()) & gt_top20) / 20.0 r1_quant = 1.0 if pred_order[0] == gt_order[0] else 0.0 r10_quant = len(set(pred_order[:10].tolist()) & set(gt_order[:10].tolist())) / 10.0 r20_quant = len(set(pred_order[:20].tolist()) & gt_top20) / 20.0 # Spearman rank correlation on top 250 ground-truth pool pool_idx = gt_order[:250] rk_p = np.argsort(np.argsort(pred_scores[pool_idx])) rk_t = np.argsort(np.argsort(exact_scores[pool_idx])) n_p = len(pool_idx) spearman_rho = float(1.0 - (6.0 * np.sum((rk_p - rk_t) ** 2)) / (n_p * (n_p ** 2 - 1))) # Biological evaluation on this query bio_metrics: Dict[str, Any] = {} hits_table: List[Dict[str, Any]] = [] gt_hits_table: List[Dict[str, Any]] = [] if dataset == "scope": q_fam = query_meta["family"] q_sfam = query_meta["superfamily"] q_fold = query_meta["fold"] order_no_self = pred_order[pred_order != cand_pos_self] if cand_pos_self >= 0 else pred_order gt_no_self = gt_order[gt_order != cand_pos_self] if cand_pos_self >= 0 else gt_order base_no_self = base_order[base_order != cand_pos_self] if cand_pos_self >= 0 else base_order is_fam_q = STATE.scope_cand_fam[order_no_self] == q_fam is_fam_gt = STATE.scope_cand_fam[gt_no_self] == q_fam is_fam_base = STATE.scope_cand_fam[base_no_self] == q_fam is_sfam_q = (STATE.scope_cand_sfam[order_no_self] == q_sfam) & (~is_fam_q) is_sfam_gt = (STATE.scope_cand_sfam[gt_no_self] == q_sfam) & (~is_fam_gt) bio_metrics = { "query_sccs": query_meta["sccs"], "family_homolog_count": int(np.sum(is_fam_q)), "superfamily_remote_count": int(np.sum(is_sfam_q)), "quant_family_ap": round(compute_average_precision(is_fam_q), 4), "fp32_family_ap": round(compute_average_precision(is_fam_gt), 4), "base_family_ap": round(compute_average_precision(is_fam_base), 4), "quant_superfamily_roc50": round(compute_roc50(is_sfam_q[STATE.scope_cand_fam[order_no_self] != q_fam]), 4), "fp32_superfamily_roc50": round(compute_roc50(is_sfam_gt[STATE.scope_cand_fam[gt_no_self] != q_fam]), 4), } fs_hits = STATE.foldseek_by_query.get(query_meta["id"], []) fs_map = {h["target_id"]: h for h in fs_hits} if cand_pos_self >= 0 and cand_pos_self < len(STATE.scope_p1_candidates): self_meta = STATE.scope_p1_candidates[cand_pos_self] self_lossy = min(round(float(pred_scores_full[cand_pos_self]), 5), 0.98500) self_hit = { "quant_rank": 1, "fp32_rank": 1, "base_1bit_rank": 1, "is_query_self": True, "id": query_meta["id"], "pdb_id": query_meta.get("pdb_id", self_meta.get("pdb_id", "")), "sccs": query_meta.get("sccs", self_meta.get("sccs", "")), "description": query_meta["description"], "organism": query_meta.get("organism", self_meta.get("organism", "")), "seq_len": query_meta.get("seq_len", self_meta.get("seq_len", 0)), "has_pdb": query_meta.get("has_pdb", self_meta.get("has_pdb", True)), "has_structure": True, "homology_tier": "Query Self (Compressed)", "quant_score": self_lossy, "fp32_score": 1.00000, "score_error": round(float(self_lossy - 1.00000), 5), "in_fp32_top20": True, "foldseek": None, "mini_trace": get_mini_ca_trace(query_meta["id"]) or get_mini_ca_trace(self_meta["id"]), "seq_preview": query_meta.get("sequence", self_meta.get("sequence", ""))[:120], } hits_table = [self_hit] gt_hits_table = [dict(self_hit)] else: hits_table = [] gt_hits_table = [] for c_pos in pred_order: if len(hits_table) >= top_k: break if c_pos == cand_pos_self: continue c_meta = STATE.scope_p1_candidates[c_pos] if c_meta["family"] == q_fam: tier = "Same Family" elif c_meta["superfamily"] == q_sfam: tier = "Same Superfamily (Remote)" elif c_meta["fold"] == q_fold: tier = "Same Fold" else: tier = "Different Fold" fs_entry = fs_map.get(c_meta["id"]) hits_table.append({ "quant_rank": len(hits_table) + 1, "fp32_rank": int(full_gt_rank_of[c_pos]), "base_1bit_rank": int(full_base_rank_of[c_pos]), "is_query_self": False, "id": c_meta["id"], "pdb_id": c_meta["pdb_id"], "sccs": c_meta["sccs"], "description": c_meta["description"], "organism": c_meta["organism"], "seq_len": c_meta["seq_len"], "has_pdb": c_meta["has_pdb"], "has_structure": c_meta.get("has_pdb", True), "homology_tier": tier, "quant_score": round(float(pred_scores_full[c_pos]), 5), "fp32_score": round(float(exact_scores_full[c_pos]), 5), "score_error": round(float(pred_scores_full[c_pos] - exact_scores_full[c_pos]), 5), "in_fp32_top20": int(gt_rank_of[c_pos]) <= 20, "foldseek": fs_entry, "mini_trace": get_mini_ca_trace(c_meta["id"]) if (len(hits_table) < 12 and c_meta["has_pdb"]) else None, "seq_preview": c_meta.get("sequence", "")[:120], }) for c_pos in gt_order: if len(gt_hits_table) >= top_k: break if c_pos == cand_pos_self: continue c_meta = STATE.scope_p1_candidates[c_pos] if c_meta["family"] == q_fam: tier = "Same Family" elif c_meta["superfamily"] == q_sfam: tier = "Same Superfamily (Remote)" elif c_meta["fold"] == q_fold: tier = "Same Fold" else: tier = "Different Fold" fs_entry = fs_map.get(c_meta["id"]) gt_hits_table.append({ "quant_rank": int(full_pred_rank_of[c_pos]), "fp32_rank": len(gt_hits_table) + 1, "base_1bit_rank": int(full_base_rank_of[c_pos]), "is_query_self": False, "id": c_meta["id"], "pdb_id": c_meta["pdb_id"], "sccs": c_meta["sccs"], "description": c_meta["description"], "organism": c_meta["organism"], "seq_len": c_meta["seq_len"], "has_pdb": c_meta["has_pdb"], "has_structure": c_meta.get("has_pdb", True), "homology_tier": tier, "quant_score": round(float(pred_scores_full[c_pos]), 5), "fp32_score": round(float(exact_scores_full[c_pos]), 5), "score_error": round(float(pred_scores_full[c_pos] - exact_scores_full[c_pos]), 5), "in_fp32_top20": int(gt_rank_of[c_pos]) <= 20, "foldseek": fs_entry, "mini_trace": get_mini_ca_trace(c_meta["id"]) if (len(gt_hits_table) < 12 and c_meta["has_pdb"]) else None, "seq_preview": c_meta.get("sequence", "")[:120], }) else: q_go = set(query_meta.get("go_terms", [])) jacc_quant = [] jacc_fp32 = [] jacc_base = [] for idx in range(min(top_k, len(pred_order))): c_pos_ns = int(pred_order[idx]) c_go_ns = set(STATE.sp_p5_candidates[c_pos_ns].get("go_terms", [])) jacc_quant.append(compute_go_jaccard(q_go, c_go_ns)) gt_c_go = set(STATE.sp_p5_candidates[int(gt_order[idx])].get("go_terms", [])) jacc_fp32.append(compute_go_jaccard(q_go, gt_c_go)) bs_c_go = set(STATE.sp_p5_candidates[int(base_order[idx])].get("go_terms", [])) jacc_base.append(compute_go_jaccard(q_go, bs_c_go)) if cand_pos_self >= 0 and cand_pos_self < len(STATE.sp_p5_candidates): self_meta = STATE.sp_p5_candidates[cand_pos_self] if cand_pos_self < len(STATE.sp_p5_candidates) else STATE.sp_entries[raw_idx] self_lossy = min(round(float(pred_scores_full[cand_pos_self]), 5), 0.98500) self_hit = { "quant_rank": 1, "fp32_rank": 1, "base_1bit_rank": 1, "is_query_self": True, "homology_tier": "Query Self (Compressed)", "id": query_meta["id"], "gene": query_meta.get("gene", self_meta.get("gene", "")), "entry_name": query_meta.get("entry_name", self_meta.get("entry_name", "")), "description": query_meta["description"], "seq_len": query_meta.get("seq_len", self_meta.get("seq_len", 0)), "go_jaccard": 1.0, "shared_go_terms": sorted(list(q_go))[:8], "total_shared_go": len(q_go), "candidate_go_count": len(q_go), "quant_score": self_lossy, "fp32_score": 1.00000, "score_error": round(float(self_lossy - 1.00000), 5), "in_fp32_top20": True, "seq_preview": query_meta.get("sequence", self_meta.get("sequence", ""))[:120], } hits_table = [self_hit] gt_hits_table = [dict(self_hit)] else: hits_table = [] gt_hits_table = [] for c_pos in pred_order: if len(hits_table) >= top_k: break if cand_pos_self >= 0 and c_pos == cand_pos_self: continue c_meta = STATE.sp_p5_candidates[c_pos] c_go = set(c_meta.get("go_terms", [])) shared_go = sorted(list(q_go & c_go)) jac = compute_go_jaccard(q_go, c_go) hits_table.append({ "quant_rank": len(hits_table) + 1, "fp32_rank": int(full_gt_rank_of[c_pos]), "base_1bit_rank": int(full_base_rank_of[c_pos]), "is_query_self": False, "homology_tier": "Functional Homolog" if jac >= 0.25 else "Shared GO Pathway" if len(shared_go) > 0 else "Candidate", "id": c_meta["id"], "gene": c_meta["gene"], "entry_name": c_meta["entry_name"], "description": c_meta["description"], "seq_len": c_meta["seq_len"], "go_jaccard": round(jac, 4), "shared_go_terms": shared_go[:8], "total_shared_go": len(shared_go), "candidate_go_count": len(c_go), "quant_score": round(float(pred_scores_full[c_pos]), 5), "fp32_score": round(float(exact_scores_full[c_pos]), 5), "score_error": round(float(pred_scores_full[c_pos] - exact_scores_full[c_pos]), 5), "in_fp32_top20": int(gt_rank_of[c_pos]) <= 20, "seq_preview": c_meta.get("sequence", "")[:120], }) for c_pos in gt_order: if len(gt_hits_table) >= top_k: break if cand_pos_self >= 0 and c_pos == cand_pos_self: continue c_meta = STATE.sp_p5_candidates[c_pos] c_go = set(c_meta.get("go_terms", [])) shared_go = sorted(list(q_go & c_go)) jac = compute_go_jaccard(q_go, c_go) gt_hits_table.append({ "quant_rank": int(full_pred_rank_of[c_pos]), "fp32_rank": len(gt_hits_table) + 1, "base_1bit_rank": int(full_base_rank_of[c_pos]), "is_query_self": False, "homology_tier": "Functional Homolog" if jac >= 0.25 else "Shared GO Pathway" if len(shared_go) > 0 else "Candidate", "id": c_meta["id"], "gene": c_meta["gene"], "entry_name": c_meta["entry_name"], "description": c_meta["description"], "seq_len": c_meta["seq_len"], "go_jaccard": round(jac, 4), "shared_go_terms": shared_go[:8], "total_shared_go": len(shared_go), "candidate_go_count": len(c_go), "quant_score": round(float(pred_scores_full[c_pos]), 5), "fp32_score": round(float(exact_scores_full[c_pos]), 5), "score_error": round(float(pred_scores_full[c_pos] - exact_scores_full[c_pos]), 5), "in_fp32_top20": int(gt_rank_of[c_pos]) <= 20, "seq_preview": c_meta.get("sequence", "")[:120], }) bio_metrics = { "query_go_count": len(q_go), "quant_top1_go_jaccard": round(float(jacc_quant[0]) if jacc_quant else 0.0, 4), "quant_top5_go_jaccard": round(float(np.mean(jacc_quant[:5])) if jacc_quant else 0.0, 4), "quant_top20_go_jaccard": round(float(np.mean(jacc_quant[:20])) if jacc_quant else 0.0, 4), "fp32_top1_go_jaccard": round(float(jacc_fp32[0]) if jacc_fp32 else 0.0, 4), "fp32_top5_go_jaccard": round(float(np.mean(jacc_fp32[:5])) if jacc_fp32 else 0.0, 4), "base_top1_go_jaccard": round(float(jacc_base[0]) if jacc_base else 0.0, 4), } # Scatter plot points (query self + top 59 homologs + 60 evenly spaced across rank distribution) sample_positions = np.concatenate([ full_gt_order[:60], full_gt_order[np.linspace(60, N - 2, 60).astype(int)], ]) scatter_points = [ { "id": (STATE.scope_p1_candidates[int(p)]["id"] if dataset == "scope" else STATE.sp_p5_candidates[int(p)]["id"]), "fp32_score": round(float(exact_scores_full[int(p)]), 5), "quant_score": round(float(pred_scores_full[int(p)]), 5), "base_1bit_score": round(float(base_scores_full[int(p)]), 5), "is_top20": (int(p) == cand_pos_self) or (int(gt_rank_of[int(p)]) <= 20), } for p in sample_positions ] codec_info = next((c for c in CODEC_CATALOG if c["id"] == codec_id), CODEC_CATALOG[0]) actual_bytes = getattr(codec, "total_bytes_per_vector", D * 4) query_out = { **query_meta, "seq_preview": query_meta.get("sequence", "")[:180], "mini_trace": get_mini_ca_trace(query_meta["id"]) if dataset == "scope" else None, } if is_four_char_pdb: pdb_up = raw_q.upper() if live_rcsb and "metadata" in live_rcsb: lm = live_rcsb["metadata"] query_out.update({ "id": pdb_up, "pdb_id": pdb_up, "chain": lm.get("chain", query_out.get("chain", "A")), "description": f"{lm.get('description', pdb_up)} [Matched via SCOPe domain {query_meta['id']} ({query_meta['sccs']})]", "organism": lm.get("organism") or query_out.get("organism", ""), "seq_len": lm.get("seq_len", query_out.get("seq_len", 0)), "sequence": lm.get("sequence", query_out.get("sequence", "")), "seq_preview": lm.get("sequence", query_out.get("sequence", ""))[:180], "mini_trace": lm.get("mini_trace") or query_out.get("mini_trace"), }) elif pdb_up != query_meta.get("pdb_id", "").upper(): query_out.update({ "id": pdb_up, "pdb_id": pdb_up, "description": f"RCSB PDB {pdb_up} — {query_meta.get('description', '')} [Matched via SCOPe domain {query_meta['id']} ({query_meta['sccs']})]", }) return { "dataset": dataset, "model_scale": model_scale, "dimension": D, "num_candidates": N, "query": query_out, "codec": { **codec_info, "actual_bytes": actual_bytes, "db_size_mb": round((N * actual_bytes) / (1024 * 1024), 2), "fp32_db_size_mb": round((N * D * 4) / (1024 * 1024), 2), }, "metrics": { "self_quant_rank": int(full_pred_rank_of[cand_pos_self]) if 0 <= cand_pos_self < N else 1, "self_quant_score": round(float(pred_scores_full[cand_pos_self]), 5) if 0 <= cand_pos_self < N else 1.0, "self_base_1bit_rank": int(full_base_rank_of[cand_pos_self]) if 0 <= cand_pos_self < N else 1, "recall_20_at_30": round(r20_30_quant, 4), "base_1bit_recall_20_at_30": round(r20_30_base, 4), "recall_1": round(r1_quant, 4), "recall_10": round(r10_quant, 4), "recall_20": round(r20_quant, 4), "inner_product_mse": float(ip_mse), "base_1bit_mse": float(base_mse), "mse_reduction_vs_1bit": round(base_mse / max(ip_mse, 1e-12), 1), "spearman_rho": round(spearman_rho, 5), "search_latency_ms": round(quant_ms, 3), "fp32_latency_ms": round(fp32_ms, 3), }, "biological": bio_metrics, "hits": hits_table, "gt_hits": gt_hits_table, "scatter_points": scatter_points, "foldseek_top20": STATE.foldseek_by_query.get(query_meta["id"], []) if dataset == "scope" else [], } def run_live_theorem_ablation( query_id: str, k: int = 4, model_scale: str = "esm2_650m", ) -> Dict[str, Any]: """ Runs a live 2x2 factorial ablation of Theorem 1 (Support-Orthogonal Bleed Elimination) and Theorem 2 (Analytical Bussgang De-Attenuation) on the chosen real SCOPe protein. """ raw_emb = STATE.get_embeddings("scope", model_scale) cand_emb = raw_emb[STATE.p1_cand_raw_ids] N, D = cand_emb.shape query_meta = STATE.resolve_scope_entry(query_id) raw_idx = query_meta["raw_idx"] cand_pos_self = STATE.scope_sid_to_p1_pos.get(query_meta["id"], 0) q_vec = raw_emb[raw_idx : raw_idx + 1] exact_scores = (q_vec @ cand_emb.T)[0].copy() valid_mask = np.ones(N, dtype=bool) if 0 <= cand_pos_self < N: valid_mask[cand_pos_self] = False exact_ranked = exact_scores.copy() if 0 <= cand_pos_self < N: exact_ranked[cand_pos_self] = -1e9 gt_order = np.argsort(-exact_ranked, kind="stable") gt_top20 = set(gt_order[:20].tolist()) sample_idx = np.concatenate([gt_order[:40], gt_order[np.linspace(40, N - 2, 40).astype(int)]]) arms_spec = [ ("naive", "1. Naive Hybrid (No Thm 1, No Thm 2)", False, False), ("thm2_only", "2. Theorem 2 Only (Bussgang Scaling Only)", False, True), ("thm1_only", "3. Theorem 1 Only (Bleed Elimination Only)", True, False), ("both", "4. Full QuantPPI (Theorem 1 + Theorem 2)", True, True), ] # Encode once (all 4 arms use the exact same wire buffer!) base_codec = DynamicTopKOutlierCodec(dimension=D, k=k, transform_name="circulant", seed=42) cache_key = ("scope", model_scale, f"ablation_wire_k{k}") if cache_key in STATE._encoded_db_cache: _, packed_bytes, _ = STATE._encoded_db_cache[cache_key] else: packed_bytes = base_codec.encode(cand_emb) STATE._encoded_db_cache[cache_key] = (base_codec, packed_bytes, 0.0) q_fam = query_meta["family"] arms_out = [] for arm_id, label, thm1, thm2 in arms_spec: arm_codec = DynamicTopKOutlierCodec( dimension=D, k=k, transform_name="circulant", seed=42, enable_bleed_elimination=thm1, enable_bussgang_scaling=thm2, ) scores = arm_codec.compute_asymmetric_scores(q_vec, packed_bytes, count=N)[0].copy() mse = float(np.mean((scores[valid_mask] - exact_scores[valid_mask]) ** 2)) mae = float(np.mean(np.abs(scores[valid_mask] - exact_scores[valid_mask]))) scores_ranked = scores.copy() if 0 <= cand_pos_self < N: scores_ranked[cand_pos_self] = -1e9 order = np.argsort(-scores_ranked, kind="stable") order_no_self = order[order != cand_pos_self] r20_30 = len(set(order[:30].tolist()) & gt_top20) / 20.0 r1 = 1.0 if order[0] == gt_order[0] else 0.0 r20 = len(set(order[:20].tolist()) & gt_top20) / 20.0 fam_ap = compute_average_precision(STATE.scope_cand_fam[order_no_self] == q_fam) arms_out.append({ "arm_id": arm_id, "label": label, "enable_bleed_elimination": thm1, "enable_bussgang_scaling": thm2, "wire_bytes": arm_codec.total_bytes_per_vector, "recall_20_at_30": round(r20_30, 4), "recall_1": round(r1, 4), "recall_20": round(r20, 4), "family_ap": round(fam_ap, 4), "inner_product_mse": float(mse), "inner_product_mae": float(mae), "scatter": [ { "fp32": round(float(exact_scores[int(idx)]), 5), "pred": round(float(scores[int(idx)]), 5), "is_top20": int(idx) in gt_top20, } for idx in sample_idx ], }) return { "query": query_meta, "model_scale": model_scale, "dimension": D, "k": k, "wire_bytes": base_codec.total_bytes_per_vector, "arms": arms_out, } class QuantPPIRequestHandler(BaseHTTPRequestHandler): def log_message(self, format: str, *args): # Concise request logging sys.stderr.write(f"[API] {self.address_string()} - {format % args}\n") def _send_json(self, payload: Any, status: int = 200): body = json.dumps(payload).encode("utf-8") self.send_response(status) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") self.send_header("Access-Control-Allow-Headers", "Content-Type") self.end_headers() self.wfile.write(body) def do_OPTIONS(self): self.send_response(204) self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") self.send_header("Access-Control-Allow-Headers", "Content-Type") self.end_headers() def do_GET(self): parsed = urllib.parse.urlparse(self.path) path = parsed.path qs = urllib.parse.parse_qs(parsed.query) def _p(name: str, default: str = "") -> str: vals = qs.get(name) return vals[0] if vals else default try: if path == "/api/overview": dataset = _p("dataset", "scope") model_scale = _p("model_scale", "esm2_650m") geom = STATE.get_corpus_geometry(dataset, model_scale) self._send_json({ "geometry": geom, "codecs": CODEC_CATALOG, "datasets": [ { "id": "scope", "name": "SCOPe 2.08 ASTRAL-40 Structural Domains", "total_sequences": len(STATE.scope_entries), "candidates": len(STATE.p1_cand_raw_ids), "canonical_queries": len(STATE.p1_query_raw_ids), "pdb_structures": 15118, "source_fasta": "data/astral-scopedom-seqres-gd-sel-gs-bib-40-2.08.fa", }, { "id": "swissprot", "name": "UniProtKB/Swiss-Prot Human Proteome", "total_sequences": len(STATE.sp_entries), "candidates": len(STATE.p5_cand_raw_ids), "canonical_queries": len(STATE.p5_query_raw_ids), "calibration_pool": len(STATE.p5_calib_raw_ids), "source_fasta": "data/swissprot_human_full.fasta", }, ], "benchmarks": STATE.benchmarks_payload, "model_scales": [ {"id": "esm2_8m", "name": "ESM-2 8M (6-layer, D=320)", "dimension": 320, "fp32_bytes": 1280}, {"id": "esm2_35m", "name": "ESM-2 35M (12-layer, D=480)", "dimension": 480, "fp32_bytes": 1920}, {"id": "esm2_150m", "name": "ESM-2 150M (30-layer, D=640)", "dimension": 640, "fp32_bytes": 2560}, {"id": "esmc_300m", "name": "ESM-C 300M (30-layer, D=960)", "dimension": 960, "fp32_bytes": 3840}, {"id": "esmc_600m", "name": "ESM-C 600M (36-layer, D=1152)", "dimension": 1152, "fp32_bytes": 4608}, {"id": "esm2_650m", "name": "ESM-2 650M (33-layer, D=1280)", "dimension": 1280, "fp32_bytes": 5120}, {"id": "esm3_1.4b", "name": "ESM-3 1.4B (48-layer, D=1536)", "dimension": 1536, "fp32_bytes": 6144}, {"id": "esm2_3b", "name": "ESM-2 3B (36-layer, D=2560)", "dimension": 2560, "fp32_bytes": 10240}, {"id": "esmc_6b", "name": "ESM-C 6B (80-layer, D=2560)", "dimension": 2560, "fp32_bytes": 10240}, {"id": "esm2_15b", "name": "ESM-2 15B (48-layer, D=5120)", "dimension": 5120, "fp32_bytes": 20480}, ], }) return if path == "/api/proteins": dataset = _p("dataset", "scope") q = _p("q", "").strip().lower() queries_only = _p("queries_only", "false").lower() in ("true", "1") limit = min(int(_p("limit", "40")), 200) offset = max(int(_p("offset", "0")), 0) if dataset == "scope": source = STATE.scope_p1_queries if queries_only else STATE.scope_p1_candidates if q: filtered = [ e for e in source if q in e["id"].lower() or q in e["pdb_id"].lower() or q in e["sccs"].lower() or q in e["description"].lower() or q in e["organism"].lower() or (len(q) >= 4 and q.upper() in e["sequence"]) ] alias_candidates: List[Dict[str, Any]] = [] if STATE.has_exact_or_alias_scope_match(q): alias_candidates.append(STATE.resolve_scope_entry(q)) if len(q) >= 2: for k_alias, target_dom in STATE.SCOPE_ALIAS_MAP.items(): if (q in k_alias or k_alias in q) and target_dom in STATE.scope_id_to_raw_idx: alias_candidates.append(STATE.scope_entries[STATE.scope_id_to_raw_idx[target_dom]]) for ah in reversed(alias_candidates): if all(x["id"] != ah["id"] for x in filtered): filtered.insert(0, ah) else: filtered = source else: source = STATE.sp_p5_queries if queries_only else (list(STATE.sp_extra_entries.values()) + STATE.sp_entries) if q: filtered = [ e for e in source if q in e["id"].lower() or q in e["gene"].lower() or q in e["entry_name"].lower() or q in e["description"].lower() or any(q in go.lower() for go in e.get("go_terms", [])) or (len(q) >= 4 and q.upper() in e["sequence"]) ] alias_candidates_sp: List[Dict[str, Any]] = [] q_up = q.upper() if q_up in STATE.SP_ALIAS_MAP and STATE.SP_ALIAS_MAP[q_up] in STATE.sp_by_acc: alias_candidates_sp.append(STATE.sp_by_acc[STATE.SP_ALIAS_MAP[q_up]]) if len(q) >= 2: for k_alias, target_acc in STATE.SP_ALIAS_MAP.items(): k_low = k_alias.lower() if (q in k_low or k_low in q) and target_acc in STATE.sp_by_acc: alias_candidates_sp.append(STATE.sp_by_acc[target_acc]) for ah in reversed(alias_candidates_sp): if all(x["id"] != ah["id"] for x in filtered): filtered.insert(0, ah) else: filtered = source slice_items = [ {k: v for k, v in item.items() if k != "sequence"} for item in filtered[offset : offset + limit] ] self._send_json({ "dataset": dataset, "total_matches": len(filtered), "items": slice_items, }) return if path == "/api/quantize": dataset = _p("dataset", "scope") model_scale = _p("model_scale", "esm2_650m") protein_id = _p("protein_id", "d6a1ia1" if dataset == "scope" else "P04637") codec_id = _p("codec_id", "track3_hybrid_k2_block224") transform_name = _p("transform_name", "circulant") custom_seq = _p("custom_sequence", "") or None res = run_single_protein_quantization_pipeline( dataset=dataset, model_scale=model_scale, protein_id=protein_id, codec_id=codec_id, transform_name=transform_name, custom_sequence=custom_seq, ) self._send_json(res) return if path == "/api/search": dataset = _p("dataset", "scope") model_scale = _p("model_scale", "esm2_650m") query_id = _p("query_id", "d1dlwa_" if dataset == "scope" else "P68871") codec_id = _p("codec_id", "track3_hybrid_k2_block224") top_k = min(int(_p("top_k", "30")), 100) fasta_sequence = _p("fasta_sequence", "").strip() or None res = run_live_search( dataset=dataset, model_scale=model_scale, query_id=query_id, codec_id=codec_id, top_k=top_k, fasta_sequence=fasta_sequence, ) self._send_json(res) return if path == "/api/ablation": query_id = _p("query_id", "d1dlwa_") k = int(_p("k", "4")) model_scale = _p("model_scale", "esm2_650m") res = run_live_theorem_ablation(query_id=query_id, k=k, model_scale=model_scale) self._send_json(res) return if path == "/api/structure": raw_dom_id = _p("id", "d6a1ia1").strip() compare_id = _p("compare_id", "").strip() def _load_structure_entry(target_raw: str) -> Optional[Dict[str, Any]]: tid = target_raw.strip() if not tid: return None if tid == "custom_fasta": if STATE._last_fasta_query: closest_scaff = STATE.match_closest_scope_by_sequence(STATE._last_fasta_query)["id"] return _load_structure_entry(closest_scaff) return _load_structure_entry("d1dlwa_") if has_pdb_structure(tid): ent_p = PDB_DIR / f"{tid}.ent" obj = parse_pdb_ca_atoms(ent_p) obj["ca_atoms"] = [dict(a) for a in obj["ca_atoms"]] if ent_p.exists(): try: obj["raw_pdb_text"] = ent_p.read_text(encoding="utf-8", errors="replace") except Exception: pass r_idx = STATE.scope_id_to_raw_idx.get(tid) obj["metadata"] = dict(STATE.scope_entries[r_idx]) if r_idx is not None else {"id": tid} return obj if re.match(r"^[0-9][A-Za-z0-9]{3}$", tid): live = fetch_live_rcsb_pdb(tid) if live: mapped = STATE.resolve_scope_entry(tid) if STATE.has_exact_or_alias_scope_match(tid) else STATE.match_closest_scope_by_sequence(live["metadata"].get("sequence", "")) meta = dict(live["metadata"]) if mapped: meta.setdefault("sccs", mapped.get("sccs", "")) meta.setdefault("class_name", mapped.get("class_name", "")) return { **live, "ca_atoms": [dict(a) for a in live["ca_atoms"]], "metadata": meta, } if tid.upper() in STATE.sp_by_acc: sp_entry = STATE.sp_by_acc[tid.upper()] scaff = STATE.match_closest_scope_by_sequence(sp_entry.get("sequence", "")) scaff_id = scaff["id"] if has_pdb_structure(scaff_id): ent_p = PDB_DIR / f"{scaff_id}.ent" obj = parse_pdb_ca_atoms(ent_p) obj["ca_atoms"] = [dict(a) for a in obj["ca_atoms"]] obj["metadata"] = { **dict(sp_entry), "sccs": scaff.get("sccs", ""), "pdb_id": scaff.get("pdb_id", ""), "description": f"{sp_entry.get('description', tid)} [Homologous crystal domain {scaff_id}]", } return obj res_entry = STATE.resolve_scope_entry(tid) resolved = STATE.scope_entries[res_entry["raw_idx"]]["id"] if "raw_idx" in res_entry else res_entry["id"] if has_pdb_structure(resolved): ent_p = PDB_DIR / f"{resolved}.ent" obj = parse_pdb_ca_atoms(ent_p) obj["ca_atoms"] = [dict(a) for a in obj["ca_atoms"]] if ent_p.exists(): try: obj["raw_pdb_text"] = ent_p.read_text(encoding="utf-8", errors="replace") except Exception: pass r_idx = STATE.scope_id_to_raw_idx.get(resolved) meta = dict(STATE.scope_entries[r_idx]) if r_idx is not None else {"id": resolved} if re.match(r"^[0-9][A-Za-z0-9]{3}$", tid): meta["id"] = tid.upper() meta["pdb_id"] = tid.upper() obj["metadata"] = meta return obj return None primary = _load_structure_entry(raw_dom_id) if not primary: self._send_json({"error": f"3D PDB structure not found for {raw_dom_id}"}, status=404) return comparison = _load_structure_entry(compare_id) if compare_id else None if comparison and primary["ca_atoms"] and comparison["ca_atoms"]: ref_xyz = np.array([[a["cx"], a["cy"], a["cz"]] for a in primary["ca_atoms"]], dtype=np.float64) mov_xyz = np.array([[a["cx"], a["cy"], a["cz"]] for a in comparison["ca_atoms"]], dtype=np.float64) aligned_xyz, rmsd = kabsch_superpose(ref_xyz, mov_xyz) for i, a in enumerate(comparison["ca_atoms"]): a["ax"] = round(float(aligned_xyz[i, 0]), 3) a["ay"] = round(float(aligned_xyz[i, 1]), 3) a["az"] = round(float(aligned_xyz[i, 2]), 3) comparison["kabsch_rmsd_angstrom"] = round(rmsd, 3) self._send_json({ "primary": primary, "comparison": comparison, }) return if path == "/api/benchmarks": self._send_json(STATE.benchmarks_payload) return # Serve static frontend from dist/ if available dist_dir = THIS_DIR / "dist" if dist_dir.exists(): rel = path.lstrip("/") or "index.html" file_path = dist_dir / rel if not file_path.exists() or file_path.is_dir(): file_path = dist_dir / "index.html" if file_path.exists(): ext = file_path.suffix.lower() mime = { ".html": "text/html; charset=utf-8", ".js": "application/javascript; charset=utf-8", ".css": "text/css; charset=utf-8", ".svg": "image/svg+xml", ".json": "application/json", ".png": "image/png", }.get(ext, "application/octet-stream") data = file_path.read_bytes() self.send_response(200) self.send_header("Content-Type", mime) self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) return self._send_json({"error": f"Not found: {path}"}, status=404) except Exception as e: import traceback traceback.print_exc() self._send_json({"error": str(e)}, status=500) def do_POST(self): parsed = urllib.parse.urlparse(self.path) path = parsed.path length = int(self.headers.get("Content-Length", "0")) raw_body = self.rfile.read(length).decode("utf-8") if length > 0 else "{}" try: body = json.loads(raw_body) if path == "/api/quantize": res = run_single_protein_quantization_pipeline( dataset=body.get("dataset", "scope"), model_scale=body.get("model_scale", "esm2_650m"), protein_id=body.get("protein_id", "d6a1ia1"), codec_id=body.get("codec_id", "track3_hybrid_k2_block224"), transform_name=body.get("transform_name", "circulant"), custom_sequence=body.get("custom_sequence"), ) self._send_json(res) return if path == "/api/search": res = run_live_search( dataset=body.get("dataset", "scope"), model_scale=body.get("model_scale", "esm2_650m"), query_id=body.get("query_id", "d1dlwa_"), codec_id=body.get("codec_id", "track3_hybrid_k2_block224"), top_k=int(body.get("top_k", 30)), fasta_sequence=body.get("fasta_sequence"), ) self._send_json(res) return self._send_json({"error": f"Unknown POST endpoint: {path}"}, status=404) except Exception as e: import traceback traceback.print_exc() self._send_json({"error": str(e)}, status=500) def main(): import argparse parser = argparse.ArgumentParser(description="QuantPPI Real-Data Web Server") parser.add_argument("--host", default=os.environ.get("HOST", "0.0.0.0")) parser.add_argument("--port", type=int, default=int(os.environ.get("PORT", "8000"))) args = parser.parse_args() server = ThreadingHTTPServer((args.host, args.port), QuantPPIRequestHandler) print(f"[QuantPPI Server] Listening on http://{args.host}:{args.port}") try: server.serve_forever() except KeyboardInterrupt: print("\n[QuantPPI Server] Shutting down.") server.server_close() if __name__ == "__main__": main()