Buckets:
| #!/usr/bin/env python3 | |
| """Shipped HAC recipe: d64 × K32. Change HEAD_DIM / BUDGET here to match other published SKUs.""" | |
| from __future__ import annotations | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import torch | |
| HEAD_DIM = 64 | |
| BUDGET = 32 | |
| MVT = 1024 | |
| MERGE_SIZE = 2 | |
| WEIGHT = 0.1 | |
| DUMP_PROTOCOL = "evie-hac-d64-k32-mvt1024-imagespan" | |
| REQUIRED_KS = (1, 5, 10) | |
| REQUIRED_METRICS = tuple( | |
| f"{metric}@{k}" | |
| for metric in ("recall", "ndcg", "mrr", "map") | |
| for k in REQUIRED_KS | |
| ) | |
| _raw_model = os.environ.get("EVIE_4_5B_DIR") or os.environ.get("MODEL_DIR") or "" | |
| MODEL_DIR = Path(_raw_model) if _raw_model else Path() | |
| SOUP_DIR = MODEL_DIR | |
| SOUP_NAME = MODEL_DIR.name if _raw_model else "EVIE-4.5B" | |
| CEILING = { | |
| "source": "EVIE-4.5B uncompressed d64", | |
| "V1 nDCG@10": 91.77, | |
| "V2 nDCG@10": 70.83, | |
| "V3 nDCG@10": 63.56, | |
| "JinaVDR nDCG@10": 80.77, | |
| "Avg4 nDCG@10": 76.73, | |
| } | |
| def w_tag(w: float = WEIGHT) -> str: | |
| return f"w{int(round(float(w) * 100)):02d}" | |
| def gib_per_million_pages(vec_per_page: float = BUDGET, dim: int = HEAD_DIM, nbytes: int = 2) -> float: | |
| return float(vec_per_page) * dim * nbytes * 1_000_000 / (2**30) | |
| def atomic_write_json(path: Path, payload) -> None: | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = path.with_name(f".{path.name}.tmp") | |
| tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") | |
| tmp.replace(path) | |
| def atomic_torch_save(path: Path, payload) -> None: | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = path.with_name(f".{path.name}.tmp") | |
| torch.save(payload, tmp) | |
| tmp.replace(path) | |
| def pack_seqs(seqs: list[torch.Tensor]) -> dict: | |
| if not seqs: | |
| raise ValueError("empty sequence list") | |
| dim = int(seqs[0].shape[-1]) | |
| lengths = [int(s.shape[0]) for s in seqs] | |
| packed = torch.empty(sum(lengths), dim, dtype=torch.float16) | |
| offsets = torch.zeros(len(seqs) + 1, dtype=torch.int64) | |
| pos = 0 | |
| for i, seq in enumerate(seqs): | |
| if seq.ndim != 2 or int(seq.shape[-1]) != dim: | |
| raise ValueError(f"seq {i} shape {tuple(seq.shape)} != [L,{dim}]") | |
| n = int(seq.shape[0]) | |
| packed[pos : pos + n] = seq.detach().to(dtype=torch.float16, device="cpu") | |
| pos += n | |
| offsets[i + 1] = pos | |
| return {"packed": packed, "offsets": offsets, "dim": dim, "n": len(seqs)} | |
| def padded_from_pack(packed: torch.Tensor, offsets: torch.Tensor) -> torch.Tensor: | |
| n = int(offsets.numel()) - 1 | |
| if n <= 0: | |
| return packed.new_zeros((0, 0, packed.shape[-1])) | |
| lengths = (offsets[1:] - offsets[:-1]).tolist() | |
| lmax = max(lengths) | |
| out = packed.new_zeros((n, lmax, packed.shape[-1])) | |
| for i, length in enumerate(lengths): | |
| a = int(offsets[i]) | |
| if length: | |
| out[i, :length] = packed[a : a + length] | |
| return out | |
| def dump_ok(task_json: Path, dump_root: Path) -> bool: | |
| try: | |
| rec = json.loads(Path(task_json).read_text(encoding="utf-8")) | |
| except Exception: | |
| return False | |
| if rec.get("dump_protocol") != DUMP_PROTOCOL: | |
| return False | |
| docs = dump_root / rec["docs"] | |
| queries = dump_root / rec["queries"] | |
| qrels = dump_root / rec["qrels"] | |
| return docs.is_file() and queries.is_file() and qrels.is_file() | |
| def load_qrels(path: Path) -> tuple[list[set[int]], list[dict[int, float]]]: | |
| rec = json.loads(Path(path).read_text(encoding="utf-8")) | |
| relevant = [set(int(x) for x in row) for row in rec["relevant"]] | |
| graded = [{int(k): float(v) for k, v in row.items()} for row in rec["graded"]] | |
| return relevant, graded | |
| def dump_qrels(path: Path, relevant, graded) -> None: | |
| payload = { | |
| "relevant": [sorted(int(x) for x in row) for row in relevant], | |
| "graded": [{str(int(k)): float(v) for k, v in row.items()} for row in graded], | |
| } | |
| atomic_write_json(path, payload) | |
| def has_complete_metrics(metrics) -> bool: | |
| if not isinstance(metrics, dict): | |
| return False | |
| return all( | |
| isinstance(metrics.get(name), (int, float)) and math.isfinite(float(metrics[name])) | |
| for name in REQUIRED_METRICS | |
| ) | |
Xet Storage Details
- Size:
- 4.17 kB
- Xet hash:
- 7bfe87c73b47286a3916ecf16ee62e5b027d3f17e1b7fd71a41fd8754c7e79ad
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.