Buckets:
| #!/usr/bin/env python3 | |
| """Encode EVIE-4.5B (active head = HEAD_DIM) once: image-span tokens + queries. | |
| Query is not pooled. Resume skips task json files whose dump_protocol still matches. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import sys | |
| import time | |
| from collections import Counter, defaultdict | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "shared")) | |
| import torch | |
| from torch.utils.data import DataLoader | |
| from common import ( | |
| DUMP_PROTOCOL, | |
| HEAD_DIM, | |
| MVT, | |
| SOUP_DIR, | |
| SOUP_NAME, | |
| atomic_torch_save, | |
| atomic_write_json, | |
| dump_ok, | |
| dump_qrels, | |
| pack_seqs, | |
| ) | |
| import eval as evie_eval # noqa: E402 | |
| from hac import image_span_from_mask # noqa: E402 | |
| from paths import forbid_venv_path # noqa: E402 | |
| from colpali_engine.models import ColQwen3_5, ColQwen3_5Processor # noqa: E402 | |
| from colpali_engine.models.qwen3_5.colqwen3_5.modeling_colqwen3_5 import ( # noqa: E402 | |
| set_active_head, | |
| ) | |
| from transformers.models.qwen3_5 import Qwen3_5Config # noqa: E402 | |
| def corpus_id_for(task: dict, max_docs: int) -> str: | |
| if task["fmt"] == "beir": | |
| payload = "beir|" + "\0".join(task["corpus"]) | |
| else: | |
| payload = "paired|" + "\0".join(task["paired"]) | |
| payload += f"|max_docs={int(max_docs)}" | |
| return hashlib.sha1(payload.encode("utf-8")).hexdigest()[:16] | |
| def _first(meta: dict, *keys, default): | |
| for key in keys: | |
| value = meta.get(key) | |
| if value is not None: | |
| return value | |
| return default | |
| def split_doc_batch(emb: torch.Tensor, batch: dict, processor) -> list[dict]: | |
| image_token_id = int(processor.image_token_id) | |
| merge = int(processor.image_processor.merge_size) | |
| attn = batch["attention_mask"].bool() | |
| ids = batch["input_ids"] | |
| thw = batch["image_grid_thw"] | |
| recs = [] | |
| for i in range(emb.shape[0]): | |
| mask = attn[i] | |
| vec = emb[i][mask].contiguous().half() | |
| tok = ids[i][mask] | |
| start, end = image_span_from_mask(tok == image_token_id) | |
| grid = [int(x) for x in thw[i].tolist()] | |
| expected = (grid[1] // merge) * (grid[2] // merge) | |
| if (end - start) != expected: | |
| raise RuntimeError( | |
| f"image tokens {end - start} != grid {grid} merge={merge} expected={expected}" | |
| ) | |
| recs.append( | |
| { | |
| "vec": vec.cpu(), | |
| "image_start": int(start), | |
| "image_end": int(end), | |
| "grid_thw": grid, | |
| } | |
| ) | |
| return recs | |
| def encode_docs(processor, model, paths, device, batch_size, num_workers, meta_keys, max_docs, dedupe_key=None): | |
| ds = evie_eval._ImageParquetDataset( | |
| paths, meta_keys=meta_keys, skip_empty_query=False, dedupe_key=dedupe_key | |
| ) | |
| loader = DataLoader( | |
| ds, | |
| batch_size=batch_size, | |
| num_workers=num_workers, | |
| collate_fn=evie_eval._collate_pil, | |
| prefetch_factor=6 if num_workers > 0 else None, | |
| persistent_workers=False, | |
| ) | |
| records = [] | |
| with ThreadPoolExecutor(max_workers=1) as pool: | |
| pending = None | |
| it = iter(loader) | |
| def submit_next(): | |
| try: | |
| imgs, metas = next(it) | |
| except StopIteration: | |
| return None | |
| return pool.submit(processor.process_images, imgs), metas | |
| pending = submit_next() | |
| while pending is not None: | |
| fut, metas = pending | |
| batch = fut.result() | |
| nxt = submit_next() | |
| batch_cpu = batch | |
| batch_gpu = {k: v.to(device, non_blocking=True) for k, v in batch.items()} | |
| emb = model(**batch_gpu).float().cpu() | |
| if emb.ndim != 3 or int(emb.shape[-1]) != HEAD_DIM: | |
| raise RuntimeError(f"doc emb shape {tuple(emb.shape)} is not [B,L,{HEAD_DIM}]") | |
| recs = split_doc_batch(emb, batch_cpu, processor) | |
| for rec, meta in zip(recs, metas): | |
| rec["meta"] = meta | |
| records.append(rec) | |
| if max_docs and len(records) >= max_docs: | |
| records = records[:max_docs] | |
| break | |
| pending = nxt | |
| return records | |
| def encode_queries(processor, model, texts, device, batch_size): | |
| seqs = [] | |
| for start in range(0, len(texts), batch_size): | |
| chunk = texts[start : start + batch_size] | |
| chunk = [t if (t and str(t).strip()) else " " for t in chunk] | |
| batch = processor.process_queries(chunk) | |
| attn = batch["attention_mask"].bool() | |
| batch_gpu = {k: v.to(device) for k, v in batch.items()} | |
| for obj in (model, getattr(model, "get_base_model", lambda: None)()): | |
| if obj is not None and hasattr(obj, "rope_deltas"): | |
| obj.rope_deltas = None | |
| emb = model(**batch_gpu).float().cpu() | |
| if emb.ndim != 3 or int(emb.shape[-1]) != HEAD_DIM: | |
| raise RuntimeError(f"query emb shape {tuple(emb.shape)} is not [B,L,{HEAD_DIM}]") | |
| for i in range(emb.shape[0]): | |
| seqs.append(emb[i][attn[i]].contiguous().half()) | |
| return seqs | |
| def save_docs(path: Path, records: list[dict], ids: list[str]) -> dict: | |
| packed = pack_seqs([r["vec"] for r in records]) | |
| payload = { | |
| "ids": ids, | |
| "packed": packed["packed"], | |
| "offsets": packed["offsets"], | |
| "image_start": torch.tensor([r["image_start"] for r in records], dtype=torch.int64), | |
| "image_end": torch.tensor([r["image_end"] for r in records], dtype=torch.int64), | |
| "grid_thw": torch.tensor([r["grid_thw"] for r in records], dtype=torch.int32), | |
| "dim": HEAD_DIM, | |
| "n": len(records), | |
| "dump_protocol": DUMP_PROTOCOL, | |
| } | |
| atomic_torch_save(path, payload) | |
| n_image = [int(r["image_end"] - r["image_start"]) for r in records] | |
| n_full = [int(r["vec"].shape[0]) for r in records] | |
| return { | |
| "n_docs": len(records), | |
| "avg_full_tokens": sum(n_full) / len(n_full), | |
| "avg_image_tokens": sum(n_image) / len(n_image), | |
| "min_image_tokens": min(n_image), | |
| "max_image_tokens": max(n_image), | |
| } | |
| def load_model(device): | |
| if not str(SOUP_DIR) or not (SOUP_DIR / "model.safetensors").is_file(): | |
| raise FileNotFoundError("set MODEL_DIR or EVIE_4_5B_DIR to an EVIE-4.5B checkpoint") | |
| model_source = str(SOUP_DIR) | |
| processor = ColQwen3_5Processor.from_pretrained(model_source, max_num_visual_tokens=MVT) | |
| config = Qwen3_5Config.from_pretrained(model_source) | |
| run_path = SOUP_DIR / "run_config.json" | |
| if run_path.is_file(): | |
| run_cfg = json.loads(run_path.read_text(encoding="utf-8")) | |
| heads = [int(d) for d in run_cfg["head_dims"]] | |
| else: | |
| heads = [int(d) for d in (getattr(config, "head_dims", None) or [])] | |
| if not heads: | |
| raise FileNotFoundError( | |
| f"{SOUP_DIR} has no head_dims; point MODEL_DIR at EVIE-4.5B" | |
| ) | |
| config.head_dims = heads | |
| config.dim = max(heads) | |
| attn = os.environ.get("EVAL_ATTN", "flash_attention_2") | |
| model = ColQwen3_5.from_pretrained( | |
| model_source, | |
| config=config, | |
| torch_dtype=torch.bfloat16, | |
| attn_implementation=attn, | |
| ) | |
| model.enable_bidirectional_attention() | |
| set_active_head(model, HEAD_DIM) | |
| model = model.to(device).eval() | |
| merge = int(processor.image_processor.merge_size) | |
| if merge != 2: | |
| raise RuntimeError(f"expected merge_size=2, got {merge}") | |
| print( | |
| f"[dump] model={SOUP_NAME} head=d{HEAD_DIM} mvt={MVT} bidir=on " | |
| f"processor={model_source} device={device}", | |
| flush=True, | |
| ) | |
| return processor, model | |
| def dump_paired(task, processor, model, device, args, dump_root: Path, cid: str): | |
| docs_path = dump_root / "docs" / f"{cid}.pt" | |
| if not docs_path.is_file(): | |
| records = encode_docs( | |
| processor, | |
| model, | |
| task["paired"], | |
| device, | |
| args.embed_batch, | |
| args.num_workers, | |
| meta_keys=["image_filename"], | |
| max_docs=args.max_docs, | |
| dedupe_key="image_filename", | |
| ) | |
| if not records: | |
| raise ValueError(f"no candidate pages for {evie_eval.task_tag(task)}") | |
| ids = [] | |
| page_to_idx = {} | |
| for i, rec in enumerate(records): | |
| key = rec["meta"].get("image_filename") | |
| if key is None: | |
| raise ValueError(f"paired candidate missing image_filename in {task['subset']}") | |
| key = str(key) | |
| if key in page_to_idx: | |
| raise ValueError(f"duplicate page was encoded twice: {key}") | |
| page_to_idx[key] = i | |
| ids.append(key) | |
| stats = save_docs(docs_path, records, ids) | |
| else: | |
| blob = torch.load(docs_path, map_location="cpu", weights_only=False) | |
| ids = list(blob["ids"]) | |
| page_to_idx = {k: i for i, k in enumerate(ids)} | |
| stats = {"n_docs": len(ids), "reused": True} | |
| queries, golds = evie_eval._paired_query_rows(task["paired"], page_to_idx) | |
| if not queries: | |
| raise ValueError(f"no non-empty queries for {task['subset']}") | |
| if args.max_queries and args.max_queries < len(queries): | |
| queries = queries[: args.max_queries] | |
| golds = golds[: args.max_queries] | |
| q_seqs = encode_queries(processor, model, queries, device, args.embed_batch) | |
| relevant = [{g} for g in golds] | |
| graded = [{g: 1.0} for g in golds] | |
| return stats, q_seqs, relevant, graded | |
| def dump_beir(task, processor, model, device, args, dump_root: Path, cid: str, corpus_cache: dict): | |
| docs_path = dump_root / "docs" / f"{cid}.pt" | |
| cache_key = (cid, args.max_docs) | |
| if corpus_cache.get("key") == cache_key: | |
| ids = corpus_cache["ids"] | |
| stats = {"n_docs": len(ids), "reused": True, "in_memory": True} | |
| elif docs_path.is_file(): | |
| blob = torch.load(docs_path, map_location="cpu", weights_only=False) | |
| ids = list(blob["ids"]) | |
| corpus_cache.clear() | |
| corpus_cache.update(key=cache_key, ids=ids) | |
| stats = {"n_docs": len(ids), "reused": True} | |
| else: | |
| records = encode_docs( | |
| processor, | |
| model, | |
| task["corpus"], | |
| device, | |
| args.embed_batch, | |
| args.num_workers, | |
| meta_keys=["corpus-id", "id"], | |
| max_docs=args.max_docs, | |
| ) | |
| ids = [str(_first(r["meta"], "corpus-id", "id", default=f"c{i}")) for i, r in enumerate(records)] | |
| if len(set(ids)) != len(ids): | |
| raise ValueError(f"duplicate corpus ids in {task['subset']}") | |
| stats = save_docs(docs_path, records, ids) | |
| corpus_cache.clear() | |
| corpus_cache.update(key=cache_key, ids=ids) | |
| cid_to_idx = {cid_: i for i, cid_ in enumerate(ids)} | |
| q_rows = evie_eval.read_parquet_rows(task["queries"]) | |
| q_texts = [str(r["text"] if "text" in r else r["query"]) for r in q_rows] | |
| q_ids = [str(r.get("id", r.get("query-id", f"q{i}"))) for i, r in enumerate(q_rows)] | |
| rel_by_q, grad_by_q = defaultdict(set), defaultdict(dict) | |
| for row in evie_eval.read_parquet_rows(task["qrels"]): | |
| qid = str(row.get("query-id", row.get("id", ""))) | |
| doc = str(row.get("corpus-id", row.get("id", ""))) | |
| score = float(row.get("score", 1.0) or 1.0) | |
| if score > 0 and doc in cid_to_idx: | |
| rel_by_q[qid].add(cid_to_idx[doc]) | |
| grad_by_q[qid][cid_to_idx[doc]] = score | |
| keep = [ | |
| i | |
| for i in range(len(q_ids)) | |
| if q_texts[i].strip() | |
| and q_texts[i].lower() != "none" | |
| and q_ids[i] in rel_by_q | |
| and rel_by_q[q_ids[i]] | |
| ] | |
| if args.max_queries and args.max_queries < len(keep): | |
| keep = keep[: args.max_queries] | |
| if not keep: | |
| raise ValueError(f"no non-empty queries with valid qrels for {task['subset']}") | |
| q_seqs = encode_queries(processor, model, [q_texts[i] for i in keep], device, args.embed_batch) | |
| relevant = [rel_by_q[q_ids[i]] for i in keep] | |
| graded = [grad_by_q[q_ids[i]] for i in keep] | |
| return stats, q_seqs, relevant, graded | |
| def parse_args(): | |
| ap = argparse.ArgumentParser(description=__doc__) | |
| ap.add_argument("--eval-root", required=True) | |
| ap.add_argument("--dump-dir", required=True) | |
| ap.add_argument("--embed-batch", type=int, default=8) | |
| ap.add_argument("--num-workers", type=int, default=8) | |
| ap.add_argument("--max-queries", type=int, default=0) | |
| ap.add_argument("--max-docs", type=int, default=0) | |
| ap.add_argument("--datasets", default="") | |
| return ap.parse_args() | |
| def main(): | |
| args = parse_args() | |
| rank, world, local = evie_eval.setup_ddp() | |
| device = torch.device(f"cuda:{local}" if torch.cuda.is_available() else "cpu") | |
| dump_root = forbid_venv_path(args.dump_dir, "dump-dir") | |
| dump_root.mkdir(parents=True, exist_ok=True) | |
| tasks = evie_eval.discover_tasks(args.eval_root) | |
| if args.datasets: | |
| want = {d.strip() for d in args.datasets.split(",") if d.strip()} | |
| tasks = [t for t in tasks if t["dataset"] in want] | |
| expected = len(tasks) | |
| if not args.datasets: | |
| want_full = {"eval": 10, "eval_v2": 4, "eval_v3": 48, "JinaVDR": 76} | |
| got = Counter(t["dataset"] for t in tasks) | |
| bad = {k: (want_full[k], got.get(k, 0)) for k in want_full if got.get(k, 0) != want_full[k]} | |
| if bad: | |
| raise RuntimeError(f"eval discovery incomplete: {bad}") | |
| pending = [] | |
| for task in tasks: | |
| slug = evie_eval.task_slug(evie_eval.task_tag(task)) | |
| marker = dump_root / "tasks" / f"{slug}.json" | |
| if dump_ok(marker, dump_root): | |
| continue | |
| pending.append(task) | |
| if world > 1: | |
| pending = evie_eval._balanced_shard(pending, world, rank) | |
| print( | |
| f"[dump rank{rank}] pending {len(pending)}/{expected} " | |
| f"(~{sum(evie_eval._task_corpus_size(t) for t in pending)} corpus rows)", | |
| flush=True, | |
| ) | |
| results = [] | |
| processor = model = None | |
| corpus_cache = {} | |
| if pending: | |
| processor, model = load_model(device) | |
| for task in pending: | |
| tag = evie_eval.task_tag(task) | |
| slug = evie_eval.task_slug(tag) | |
| cid = corpus_id_for(task, args.max_docs) | |
| t0 = time.time() | |
| try: | |
| if task["fmt"] == "paired": | |
| stats, q_seqs, relevant, graded = dump_paired( | |
| task, processor, model, device, args, dump_root, cid | |
| ) | |
| else: | |
| stats, q_seqs, relevant, graded = dump_beir( | |
| task, processor, model, device, args, dump_root, cid, corpus_cache | |
| ) | |
| q_path = dump_root / "queries" / f"{slug}.pt" | |
| r_path = dump_root / "qrels" / f"{slug}.json" | |
| atomic_torch_save(q_path, {**pack_seqs(q_seqs), "dump_protocol": DUMP_PROTOCOL}) | |
| dump_qrels(r_path, relevant, graded) | |
| rec = { | |
| "task": tag, | |
| "slug": slug, | |
| "dataset": task["dataset"], | |
| "subset": task["subset"], | |
| "lang": task.get("lang"), | |
| "fmt": task["fmt"], | |
| "corpus_id": cid, | |
| "docs": f"docs/{cid}.pt", | |
| "queries": f"queries/{slug}.pt", | |
| "qrels": f"qrels/{slug}.json", | |
| "n_docs": int(stats["n_docs"]), | |
| "n_queries": len(q_seqs), | |
| "stats": stats, | |
| "seconds": round(time.time() - t0, 1), | |
| "dump_protocol": DUMP_PROTOCOL, | |
| "head_dim": HEAD_DIM, | |
| "max_visual_tokens": MVT, | |
| "bidirectional_attention": "on", | |
| "weights": SOUP_NAME, | |
| "alpha": 0.5, | |
| } | |
| atomic_write_json(dump_root / "tasks" / f"{slug}.json", rec) | |
| results.append(rec) | |
| print( | |
| f"[dump rank{rank}] {tag}: {rec['n_queries']}q / {rec['n_docs']}d " | |
| f"{rec['seconds']}s", | |
| flush=True, | |
| ) | |
| except Exception as exc: | |
| import traceback | |
| traceback.print_exc() | |
| results.append({"task": tag, "slug": slug, "error": repr(exc)}) | |
| print(f"[dump rank{rank}] {tag} FAILED: {exc!r}", flush=True) | |
| rank_path = dump_root / f"rank_{rank}.json" | |
| atomic_write_json(rank_path, results) | |
| if torch.cuda.is_available() and model is not None: | |
| peak = torch.cuda.max_memory_allocated(device) / 1024**3 | |
| print(f"[dump rank{rank}] peak VRAM {peak:.2f} GiB", flush=True) | |
| if world > 1: | |
| torch.distributed.destroy_process_group() | |
| if rank != 0: | |
| return | |
| deadline = time.monotonic() + int(os.environ.get("EVAL_FINALIZE_TIMEOUT_S", "21600")) | |
| missing = list(range(world)) | |
| last = 0.0 | |
| while missing: | |
| missing = [] | |
| for r in range(world): | |
| p = dump_root / f"rank_{r}.json" | |
| try: | |
| json.loads(p.read_text(encoding="utf-8")) | |
| except Exception: | |
| missing.append(r) | |
| if not missing: | |
| break | |
| now = time.monotonic() | |
| if now >= deadline: | |
| print(f"[dump] finalize timeout; missing ranks {missing}", flush=True) | |
| break | |
| if now - last >= 60: | |
| print(f"[dump] waiting for ranks {missing}", flush=True) | |
| last = now | |
| time.sleep(2) | |
| ok = [] | |
| failed = [] | |
| for task in tasks: | |
| slug = evie_eval.task_slug(evie_eval.task_tag(task)) | |
| marker = dump_root / "tasks" / f"{slug}.json" | |
| if dump_ok(marker, dump_root): | |
| ok.append(json.loads(marker.read_text(encoding="utf-8"))) | |
| else: | |
| failed.append(evie_eval.task_tag(task)) | |
| unique_docs = sorted({rec["corpus_id"] for rec in ok}) | |
| done = { | |
| "dump_protocol": DUMP_PROTOCOL, | |
| "status": "complete" if not failed else "partial", | |
| "expected_tasks": expected, | |
| "completed_tasks": len(ok), | |
| "failed_or_missing": failed, | |
| "unique_corpora": len(unique_docs), | |
| "n_docs_sum_over_tasks": sum(rec["n_docs"] for rec in ok), | |
| "n_queries_sum": sum(rec["n_queries"] for rec in ok), | |
| "head_dim": HEAD_DIM, | |
| "max_visual_tokens": MVT, | |
| "weights": SOUP_NAME, | |
| "alpha": 0.5, | |
| "truncated": bool(args.max_docs or args.max_queries or args.datasets), | |
| "datasets": args.datasets or None, | |
| "eval_root": str(Path(args.eval_root).resolve()), | |
| } | |
| atomic_write_json(dump_root / "DONE.json", done) | |
| print(f"[dump] {done['status']} {done['completed_tasks']}/{expected} -> {dump_root / 'DONE.json'}", flush=True) | |
| if failed: | |
| for tag in failed[:20]: | |
| print(f" - MISS {tag}", flush=True) | |
| raise SystemExit(2) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 19 kB
- Xet hash:
- 8cbd122d3f253baa7de959a7ec3b0a2e95cf05c149ebbfb0474a68ff1f95e0dc
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.