#!/usr/bin/env python3 """Priority-0 audit for the RAVEL/CLARA revision. This script intentionally avoids torch/transformers so it can run before the training environment is installed. It audits: - local dataset layout and split manifests; - HFM provenance from the Hugging Face dataset repository; - duplicate/leakage indicators for HFM; - static architecture and loss/gradient-flow issues visible from code. """ from __future__ import annotations import ast import csv import hashlib import json import os import platform import random import re import statistics import subprocess import sys import textwrap import urllib.request from collections import Counter, defaultdict from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Tuple PROJECT_ROOT = Path(__file__).resolve().parents[1] OUT_ROOT = PROJECT_ROOT / "ravel_revision_results" LABELS = {"positive", "neutral", "negative"} LABEL_NAME_TO_ID = {"positive": 0, "neutral": 1, "negative": 2} @dataclass class MVSAAuditSample: dataset: str sample_id: str image_path: str text_path: str label: str text_labels: List[str] image_labels: List[str] text_majority: str image_majority: str @dataclass class HFMAuditSample: sample_id: str split: str text: str label: str image_path: str raw_record_len: int def ensure_dirs() -> None: for rel in [ "environment", "data_audit", "architecture", "configs/main", "configs/ablation", "configs/disagreement", "configs/calibration", "configs/stress_test", "configs/hyperparameters", "configs/lvlm", "aggregate_results", "predictions/mvsa", "predictions/hfm", "training_logs", "attention_maps", "error_analysis", "figures", "pilot", ]: (OUT_ROOT / rel).mkdir(parents=True, exist_ok=True) def write_text(path: Path, content: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(textwrap.dedent(content).strip() + "\n", encoding="utf-8") def write_csv(path: Path, rows: Iterable[Dict[str, Any]], fieldnames: List[str]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") writer.writeheader() for row in rows: writer.writerow(row) def run_text(cmd: List[str], cwd: Path = PROJECT_ROOT) -> str: try: proc = subprocess.run( cmd, cwd=str(cwd), text=True, capture_output=True, check=False, ) out = (proc.stdout or "") + (proc.stderr or "") return out.strip() except Exception as exc: return f"ERROR running {' '.join(cmd)}: {exc}" def fetch_hf_metadata() -> Dict[str, Any]: url = "https://huggingface.co/api/datasets/minhy112/dataset_phuonglam" try: with urllib.request.urlopen(url, timeout=20) as resp: return json.loads(resp.read().decode("utf-8")) except Exception as exc: return {"error": str(exc), "url": url} def parse_pair(value: str) -> Optional[Tuple[str, str]]: parts = [token.strip().lower() for token in value.split(",")] if len(parts) != 2: return None if parts[0] not in LABELS or parts[1] not in LABELS: return None return parts[0], parts[1] def majority(labels: List[str]) -> str: return Counter(labels).most_common(1)[0][0] def fuse_modal_labels(text_label: str, image_label: str) -> Optional[str]: if {text_label, image_label} == {"positive", "negative"}: return None if text_label == "positive" or image_label == "positive": return "positive" if text_label == "negative" or image_label == "negative": return "negative" return "neutral" def find_image(data_dir: Path, sample_id: str) -> Optional[Path]: for ext in [".jpg", ".JPG", ".jpeg", ".JPEG", ".png", ".PNG", ".bmp", ".BMP"]: path = data_dir / f"{sample_id}{ext}" if path.exists(): return path return None def load_mvsa_single(root: Path) -> List[MVSAAuditSample]: data_dir = root / "data" label_file = root / "labelResultAll.txt" lines = label_file.read_text(encoding="utf-8", errors="ignore").splitlines() samples: List[MVSAAuditSample] = [] for line in lines[1:]: parts = line.strip().split("\t") if len(parts) < 2: continue sample_id = parts[0].strip() pair = parse_pair(parts[1]) if pair is None: continue t1, i1 = pair label = fuse_modal_labels(t1, i1) if label is None: continue text_path = data_dir / f"{sample_id}.txt" image_path = find_image(data_dir, sample_id) if not text_path.exists() or image_path is None: continue samples.append( MVSAAuditSample( dataset="MVSA-Single", sample_id=sample_id, image_path=str(image_path), text_path=str(text_path), label=label, text_labels=[t1], image_labels=[i1], text_majority=t1, image_majority=i1, ) ) return samples def load_mvsa_multiple(root: Path, paper_exact_counts: bool = True) -> List[MVSAAuditSample]: data_dir = root / "data" label_file = root / "labelResultAll.txt" lines = label_file.read_text(encoding="utf-8", errors="ignore").splitlines() samples: List[MVSAAuditSample] = [] for line in lines[1:]: parts = line.strip().split("\t") if len(parts) < 4: continue sample_id = parts[0].strip() parsed = [parse_pair(parts[idx]) for idx in (1, 2, 3)] if any(item is None for item in parsed): continue pairs = [item for item in parsed if item is not None] text_labels = [pair[0] for pair in pairs] image_labels = [pair[1] for pair in pairs] if len(set(text_labels)) == 3 or len(set(image_labels)) == 3: continue text_majority = majority(text_labels) image_majority = majority(image_labels) label = fuse_modal_labels(text_majority, image_majority) if label is None: continue text_path = data_dir / f"{sample_id}.txt" image_path = find_image(data_dir, sample_id) if not text_path.exists() or image_path is None: continue samples.append( MVSAAuditSample( dataset="MVSA-Multiple", sample_id=sample_id, image_path=str(image_path), text_path=str(text_path), label=label, text_labels=text_labels, image_labels=image_labels, text_majority=text_majority, image_majority=image_majority, ) ) if paper_exact_counts: targets = {"positive": 11318, "neutral": 4408, "negative": 1298} kept: List[MVSAAuditSample] = [] by_label: Dict[str, List[MVSAAuditSample]] = defaultdict(list) for sample in samples: by_label[sample.label].append(sample) for label, target in targets.items(): bucket = sorted(by_label[label], key=lambda item: int(item.sample_id)) kept.extend(bucket[: min(target, len(bucket))]) return kept return samples def split_mvsa( samples: List[MVSAAuditSample], train_ratio: float = 0.8, val_ratio: float = 0.1, seed: int = 42, ) -> Dict[str, List[MVSAAuditSample]]: random.seed(seed) groups: Dict[str, List[MVSAAuditSample]] = {"positive": [], "neutral": [], "negative": []} for sample in samples: groups[sample.label].append(sample) split_map = {"train": [], "val": [], "test": []} for group in groups.values(): random.shuffle(group) n = len(group) n_train = int(n * train_ratio) n_val = int(n * val_ratio) split_map["train"].extend(group[:n_train]) split_map["val"].extend(group[n_train : n_train + n_val]) split_map["test"].extend(group[n_train + n_val :]) random.shuffle(split_map["train"]) random.shuffle(split_map["val"]) random.shuffle(split_map["test"]) return split_map def normalize_text(text: str) -> str: return re.sub(r"\s+", " ", text.strip().lower()) def hfm_image_dirs(root: Path, split: str) -> List[Path]: dirs = [root / split / "image"] if split == "val": dirs.append(root / "valid" / "image") dirs.append(root / "image") return dirs def find_hfm_image(root: Path, split: str, sample_id: str) -> Optional[Path]: for image_dir in hfm_image_dirs(root, split): if not image_dir.exists(): continue for ext in [".jpg", ".JPG", ".jpeg", ".JPEG", ".png", ".PNG", ".bmp"]: path = image_dir / f"{sample_id}{ext}" if path.exists(): return path return None def load_hfm(root: Path) -> Tuple[List[HFMAuditSample], Dict[str, int], Dict[str, int]]: text_dir = root / "text" loaded: List[HFMAuditSample] = [] raw_counts: Dict[str, int] = {} found_counts: Dict[str, int] = {} for filename, split in [("train.txt", "train"), ("val.txt", "val"), ("test.txt", "test")]: raw_counts[split] = 0 found_counts[split] = 0 file_path = text_dir / filename if not file_path.exists(): continue for line in file_path.read_text(encoding="utf-8", errors="ignore").splitlines(): line = line.strip() if not line: continue try: record = ast.literal_eval(line) except Exception: continue if not isinstance(record, (list, tuple)) or len(record) < 3: continue raw_counts[split] += 1 sample_id = str(record[0]).strip() text = str(record[1]).strip() label = str(record[2]).strip() image_path = find_hfm_image(root, split, sample_id) if image_path is None: continue found_counts[split] += 1 loaded.append( HFMAuditSample( sample_id=sample_id, split=split, text=text, label=label, image_path=str(image_path), raw_record_len=len(record), ) ) return loaded, raw_counts, found_counts def sha256_file(path: Path) -> str: h = hashlib.sha256() with path.open("rb") as f: while True: chunk = f.read(1024 * 1024) if not chunk: break h.update(chunk) return h.hexdigest() def summarize_counter(counter: Counter) -> str: return json.dumps(dict(sorted(counter.items())), ensure_ascii=False) def write_mvsa_manifest(filename: str, split_map: Dict[str, List[MVSAAuditSample]]) -> None: rows: List[Dict[str, Any]] = [] for split, samples in split_map.items(): for sample in samples: rows.append( { "dataset": sample.dataset, "split": split, "sample_id": sample.sample_id, "label": sample.label, "label_id": LABEL_NAME_TO_ID[sample.label], "image_path": sample.image_path, "text_path": sample.text_path, "number_of_images": 1, "text_labels": "|".join(sample.text_labels), "image_labels": "|".join(sample.image_labels), "text_majority": sample.text_majority, "image_majority": sample.image_majority, } ) write_csv( OUT_ROOT / "data_audit" / filename, rows, [ "dataset", "split", "sample_id", "label", "label_id", "image_path", "text_path", "number_of_images", "text_labels", "image_labels", "text_majority", "image_majority", ], ) def write_hfm_manifest(samples: List[HFMAuditSample]) -> None: rows = [] for sample in samples: label_id = _hfm_binary_label(sample.label) rows.append( { "dataset": "HFM", "split": sample.split, "sample_id": sample.sample_id, "raw_label": sample.label, "label_id": "" if label_id is None else label_id, "label_name": "" if label_id is None else _hfm_label_name(label_id), "label_mapping_status": "binary_hfm_mapping" if label_id is not None else "invalid_label", "image_path": sample.image_path, "text": sample.text, "text_norm": normalize_text(sample.text), "number_of_images": 1, "raw_record_len": sample.raw_record_len, } ) write_csv( OUT_ROOT / "data_audit" / "split_manifest_hfm.csv", rows, [ "dataset", "split", "sample_id", "raw_label", "label_id", "label_name", "label_mapping_status", "image_path", "text", "text_norm", "number_of_images", "raw_record_len", ], ) def dataset_statistics_rows( mvsa_single: List[MVSAAuditSample], mvsa_multiple: List[MVSAAuditSample], hfm_samples: List[HFMAuditSample], hfm_raw_counts: Dict[str, int], hfm_found_counts: Dict[str, int], ) -> List[Dict[str, Any]]: rows: List[Dict[str, Any]] = [] for dataset, samples in [ ("MVSA-Single", mvsa_single), ("MVSA-Multiple", mvsa_multiple), ]: counts = Counter(sample.label for sample in samples) rows.append( { "dataset": dataset, "loaded_samples": len(samples), "raw_text_rows": "", "train_samples_with_images": "", "val_samples_with_images": "", "test_samples_with_images": "", "positive_or_label0": counts.get("positive", 0), "neutral_or_label1": counts.get("neutral", 0), "negative_or_label2": counts.get("negative", 0), "mean_images_per_sample": "1.0000", "median_images_per_sample": "1", "max_images_per_sample": "1", "samples_with_gt_1_image": "0", "aggregation_in_current_code": "single image; CLIP tokens mean-pooled before fusion", } ) hfm_counts = Counter(sample.label for sample in hfm_samples) rows.append( { "dataset": "HFM", "loaded_samples": len(hfm_samples), "raw_text_rows": sum(hfm_raw_counts.values()), "train_samples_with_images": hfm_found_counts.get("train", 0), "val_samples_with_images": hfm_found_counts.get("val", 0), "test_samples_with_images": hfm_found_counts.get("test", 0), "positive_or_label0": hfm_counts.get("0", 0), "neutral_or_label1": hfm_counts.get("1", 0), "negative_or_label2": hfm_counts.get("2", 0), "mean_images_per_sample": "1.0000", "median_images_per_sample": "1", "max_images_per_sample": "1", "samples_with_gt_1_image": "0", "aggregation_in_current_code": "single image; CLIP tokens mean-pooled before fusion", } ) return rows def write_multi_image_table( mvsa_single: List[MVSAAuditSample], mvsa_multiple: List[MVSAAuditSample], hfm_samples: List[HFMAuditSample], ) -> None: rows = [ { "Dataset": "MVSA-Single", "Samples": len(mvsa_single), "Mean images/sample": "1.0000", "Median": 1, "Max": 1, "Samples with >1 image": 0, "Aggregation": "none; one image file per sample; model mean-pools CLIP tokens", }, { "Dataset": "MVSA-Multiple", "Samples": len(mvsa_multiple), "Mean images/sample": "1.0000", "Median": 1, "Max": 1, "Samples with >1 image": 0, "Aggregation": "none; one image file per sample; MVSA-Multiple refers annotations, not multi-image posts", }, { "Dataset": "HFM", "Samples": len(hfm_samples), "Mean images/sample": "1.0000", "Median": 1, "Max": 1, "Samples with >1 image": 0, "Aggregation": "none; one image file per loaded sample; model mean-pools CLIP tokens", }, ] write_csv( OUT_ROOT / "data_audit" / "multi_image_handling.csv", rows, [ "Dataset", "Samples", "Mean images/sample", "Median", "Max", "Samples with >1 image", "Aggregation", ], ) def write_hfm_duplicate_reports(samples: List[HFMAuditSample]) -> None: duplicate_rows: List[Dict[str, Any]] = [] leakage_rows: List[Dict[str, Any]] = [] def add_duplicate(kind: str, key: str, grouped: List[HFMAuditSample]) -> None: splits = sorted({sample.split for sample in grouped}) if len(grouped) <= 1: return duplicate_rows.append( { "dataset": "HFM", "duplicate_type": kind, "key": key, "total_occurrences": len(grouped), "splits": "|".join(splits), "cross_split": int(len(splits) > 1), "sample_ids": "|".join(sample.sample_id for sample in grouped[:50]), "labels": "|".join(sample.label for sample in grouped[:50]), "paths": "|".join(sample.image_path for sample in grouped[:20]), } ) by_id: Dict[str, List[HFMAuditSample]] = defaultdict(list) by_text: Dict[str, List[HFMAuditSample]] = defaultdict(list) for sample in samples: by_id[sample.sample_id].append(sample) by_text[normalize_text(sample.text)].append(sample) for key, grouped in by_id.items(): add_duplicate("image_id", key, grouped) for key, grouped in by_text.items(): add_duplicate("exact_normalized_text", hashlib.sha1(key.encode("utf-8")).hexdigest(), grouped) hash_cache: Dict[str, str] = {} by_hash: Dict[str, List[HFMAuditSample]] = defaultdict(list) for sample in samples: path = sample.image_path if path not in hash_cache: hash_cache[path] = sha256_file(Path(path)) by_hash[hash_cache[path]].append(sample) for key, grouped in by_hash.items(): add_duplicate("image_sha256", key, grouped) write_csv( OUT_ROOT / "data_audit" / "duplicate_report.csv", duplicate_rows, [ "dataset", "duplicate_type", "key", "total_occurrences", "splits", "cross_split", "sample_ids", "labels", "paths", ], ) for kind in ["image_id", "exact_normalized_text", "image_sha256"]: rows = [row for row in duplicate_rows if row["duplicate_type"] == kind] cross = [row for row in rows if str(row["cross_split"]) == "1"] severity = "high" if kind in {"image_id", "image_sha256"} and cross else ("medium" if cross else "none") leakage_rows.append( { "dataset": "HFM", "check": kind, "finding": f"{len(rows)} duplicate keys; {len(cross)} cross-split duplicate keys", "severity": severity, "details": "See duplicate_report.csv", } ) train_ids = {sample.sample_id for sample in samples if sample.split == "train"} val_ids = {sample.sample_id for sample in samples if sample.split == "val"} test_ids = {sample.sample_id for sample in samples if sample.split == "test"} leakage_rows.append( { "dataset": "HFM", "check": "split_id_overlap", "finding": json.dumps( { "train_val": len(train_ids & val_ids), "train_test": len(train_ids & test_ids), "val_test": len(val_ids & test_ids), } ), "severity": "high" if (train_ids & val_ids or train_ids & test_ids or val_ids & test_ids) else "none", "details": "Overlap is computed on raw image/text sample IDs after filtering to samples with images.", } ) write_csv( OUT_ROOT / "data_audit" / "leakage_report.csv", leakage_rows, ["dataset", "check", "finding", "severity", "details"], ) class UnionFind: def __init__(self, n: int): self.parent = list(range(n)) self.rank = [0] * n def find(self, x: int) -> int: while self.parent[x] != x: self.parent[x] = self.parent[self.parent[x]] x = self.parent[x] return x def union(self, a: int, b: int) -> None: ra = self.find(a) rb = self.find(b) if ra == rb: return if self.rank[ra] < self.rank[rb]: ra, rb = rb, ra self.parent[rb] = ra if self.rank[ra] == self.rank[rb]: self.rank[ra] += 1 def _hfm_binary_label(raw_label: str) -> Optional[int]: value = str(raw_label).strip().lower() if value in {"0", "non-hateful", "non_hateful", "non hateful", "not hateful", "benign"}: return 0 if value in {"1", "hateful", "hate"}: return 1 return None def _hfm_label_name(label_id: int) -> str: return "Non-hateful" if int(label_id) == 0 else "Hateful" def _cross_split_key_count(key_to_samples: Dict[str, List[HFMAuditSample]]) -> int: return sum(1 for grouped in key_to_samples.values() if len({sample.split for sample in grouped}) > 1) def write_hfm_deleaked_split(samples: List[HFMAuditSample], seed: int = 42) -> None: """Build duplicate-connected HFM components and a group-stratified split.""" n = len(samples) uf = UnionFind(n) text_to_indices: Dict[str, List[int]] = defaultdict(list) hash_to_indices: Dict[str, List[int]] = defaultdict(list) image_hashes: Dict[int, str] = {} for idx, sample in enumerate(samples): text_to_indices[normalize_text(sample.text)].append(idx) image_hash = sha256_file(Path(sample.image_path)) image_hashes[idx] = image_hash hash_to_indices[image_hash].append(idx) for grouped in list(text_to_indices.values()) + list(hash_to_indices.values()): if len(grouped) <= 1: continue first = grouped[0] for other in grouped[1:]: uf.union(first, other) components_by_root: Dict[int, List[int]] = defaultdict(list) for idx in range(n): components_by_root[uf.find(idx)].append(idx) components = list(components_by_root.values()) total_label_counts = Counter() for sample in samples: label_id = _hfm_binary_label(sample.label) if label_id is not None: total_label_counts[label_id] += 1 ratios = {"train": 0.8, "val": 0.1, "test": 0.1} desired_total = {split: ratios[split] * len(samples) for split in ratios} desired_label = { split: {label: ratios[split] * count for label, count in total_label_counts.items()} for split in ratios } component_label_counts: List[Counter] = [] for indices in components: comp_counts = Counter() for idx in indices: label_id = _hfm_binary_label(samples[idx].label) if label_id is not None: comp_counts[label_id] += 1 component_label_counts.append(comp_counts) def objective(sizes: Counter, counts: Dict[str, Counter]) -> float: score = 0.0 for split in ["train", "val", "test"]: score += 4.0 * ( (sizes[split] - desired_total[split]) / max(1.0, desired_total[split]) ) ** 2 for label in sorted(total_label_counts): score += 10.0 * ( (counts[split][label] - desired_label[split][label]) / max(1.0, desired_label[split][label]) ) ** 2 return score best_component_split: Dict[int, str] = {} best_split_sizes: Counter = Counter() best_score = float("inf") rng = random.Random(seed) attempts = 96 component_ids = list(range(len(components))) for _attempt in range(attempts): ordered = sorted( component_ids, key=lambda cid: (-len(components[cid]), rng.random()), ) split_counts = {split: Counter() for split in ratios} split_sizes = Counter() candidate_split: Dict[int, str] = {} for component_id in ordered: indices = components[component_id] comp_counts = component_label_counts[component_id] chosen_split = "train" chosen_score = float("inf") for split in ["train", "val", "test"]: projected_sizes = Counter(split_sizes) projected_counts = {name: Counter(values) for name, values in split_counts.items()} projected_sizes[split] += len(indices) projected_counts[split].update(comp_counts) score = objective(projected_sizes, projected_counts) if score < chosen_score: chosen_score = score chosen_split = split candidate_split[component_id] = chosen_split split_sizes[chosen_split] += len(indices) split_counts[chosen_split].update(comp_counts) score = objective(split_sizes, split_counts) if score < best_score: best_score = score best_component_split = candidate_split best_split_sizes = Counter(split_sizes) component_split = best_component_split split_sizes = best_split_sizes component_rows: List[Dict[str, Any]] = [] manifest_rows: List[Dict[str, Any]] = [] for component_id, indices in enumerate(components): deleaked_split = component_split[component_id] component_splits = sorted({samples[idx].split for idx in indices}) component_labels = Counter() for idx in indices: label_id = _hfm_binary_label(samples[idx].label) if label_id is not None: component_labels[label_id] += 1 for idx in indices: sample = samples[idx] label_id = _hfm_binary_label(sample.label) if label_id is None: continue row = { "dataset": "HFM-DeLeak", "duplicate_component_id": component_id, "component_size": len(indices), "component_original_splits": "|".join(component_splits), "component_label_counts": json.dumps(dict(component_labels), sort_keys=True), "original_split": sample.split, "split": deleaked_split, "sample_id": sample.sample_id, "label_id": label_id, "label_name": _hfm_label_name(label_id), "raw_label": sample.label, "image_sha256": image_hashes[idx], "image_path": sample.image_path, "text_norm": normalize_text(sample.text), "text": sample.text, } component_rows.append(row) manifest_rows.append(row) fieldnames = [ "dataset", "duplicate_component_id", "component_size", "component_original_splits", "component_label_counts", "original_split", "split", "sample_id", "label_id", "label_name", "raw_label", "image_sha256", "image_path", "text_norm", "text", ] write_csv(OUT_ROOT / "data_audit" / "hfm_duplicate_components.csv", component_rows, fieldnames) write_csv(OUT_ROOT / "data_audit" / "hfm_split_manifest_deleaked.csv", manifest_rows, fieldnames) def stats_rows(rows: List[Dict[str, Any]], split_key: str) -> List[Dict[str, Any]]: out: List[Dict[str, Any]] = [] for split in ["train", "val", "test"]: split_rows = [row for row in rows if row[split_key] == split] counts = Counter(int(row["label_id"]) for row in split_rows) out.append( { "protocol": "HFM-DeLeak" if split_key == "split" else "Existing", "split": split, "samples": len(split_rows), "non_hateful": counts.get(0, 0), "hateful": counts.get(1, 0), } ) return out write_csv( OUT_ROOT / "data_audit" / "hfm_existing_split_statistics.csv", stats_rows(component_rows, "original_split"), ["protocol", "split", "samples", "non_hateful", "hateful"], ) write_csv( OUT_ROOT / "data_audit" / "hfm_deleaked_split_statistics.csv", stats_rows(component_rows, "split"), ["protocol", "split", "samples", "non_hateful", "hateful"], ) existing_text_cross = _cross_split_key_count( {key: [samples[idx] for idx in indices] for key, indices in text_to_indices.items()} ) existing_image_cross = _cross_split_key_count( {key: [samples[idx] for idx in indices] for key, indices in hash_to_indices.items()} ) comparison_rows = [ { "Protocol": "Existing", "Train": sum(1 for sample in samples if sample.split == "train"), "Val": sum(1 for sample in samples if sample.split == "val"), "Test": sum(1 for sample in samples if sample.split == "test"), "Cross-split text duplicates": existing_text_cross, "Cross-split image duplicates": existing_image_cross, }, { "Protocol": "HFM-DeLeak", "Train": split_sizes["train"], "Val": split_sizes["val"], "Test": split_sizes["test"], "Cross-split text duplicates": 0, "Cross-split image duplicates": 0, }, ] write_csv( OUT_ROOT / "data_audit" / "hfm_split_protocol_comparison.csv", comparison_rows, [ "Protocol", "Train", "Val", "Test", "Cross-split text duplicates", "Cross-split image duplicates", ], ) write_csv( OUT_ROOT / "data_audit" / "hfm_label_mapping.csv", [ {"label_id": 0, "label_name": "Non-hateful", "positive_class_for_auroc": 0}, {"label_id": 1, "label_name": "Hateful", "positive_class_for_auroc": 1}, ], ["label_id", "label_name", "positive_class_for_auroc"], ) write_text( OUT_ROOT / "data_audit" / "hfm_provenance.md", f""" # HFM provenance and split audit ## Local source Local data was downloaded from: `https://huggingface.co/datasets/minhy112/dataset_phuonglam/tree/main` Local HFM extraction root: `data/HFM` Text files used by the current loader: - `data/HFM/text/train.txt` - `data/HFM/text/val.txt` - `data/HFM/text/test.txt` ## Current sample count Records with image files found by the loader: {len(samples)} Existing split counts: ```json {json.dumps({split: sum(1 for sample in samples if sample.split == split) for split in ["train", "val", "test"]}, indent=2)} ``` ## Label mapping The revised HFM pipeline uses a binary output space: - `0`: Non-hateful - `1`: Hateful The AUROC positive class is `1` (`Hateful`). ## Provenance limitation The downloaded Hugging Face repository does not include a README, citation file, official-split statement, augmentation manifest, or upstream dataset hash manifest. Unless a primary upstream source is added later, the paper should describe this corpus as the local HFM-derived corpus from the cited repository, not as proof that the 24,635-record split is the original official Hateful Memes Challenge corpus. ## Duplicate/leakage handling A duplicate graph was built over samples. Edges connect samples when they have the same exact normalized text or the same image SHA256 hash. Connected components were split as groups to produce `HFM-DeLeak`. Output files: - `hfm_duplicate_components.csv` - `hfm_split_manifest_deleaked.csv` - `hfm_existing_split_statistics.csv` - `hfm_deleaked_split_statistics.csv` - `hfm_split_protocol_comparison.csv` """, ) def write_environment_files() -> None: write_text( OUT_ROOT / "environment" / "hardware.txt", f""" platform: {platform.platform()} python: {sys.version.replace(os.linesep, " ")} processor: {platform.processor()} uname: {run_text(["uname", "-a"])} lscpu: {run_text(["lscpu"])} nvidia-smi: {run_text(["nvidia-smi"])} """, ) write_text(OUT_ROOT / "environment" / "pip_freeze.txt", run_text([sys.executable, "-m", "pip", "freeze"])) write_text(OUT_ROOT / "environment" / "git_commit.txt", run_text(["git", "rev-parse", "HEAD"])) write_text( OUT_ROOT / "environment" / "environment.yml", """ name: ravel-revision-audit note: Current execution environment lacks torch/transformers/peft/Pillow/sklearn. install_project_requirements: pip install -r requirements.txt runtime_probe_after_install: python scripts/probe_runtime_architecture.py --pipeline mvsa_multiple --output-dir ravel_revision_results/architecture """, ) def write_provenance(hf_meta: Dict[str, Any]) -> None: local_files = [] download_dir = PROJECT_ROOT / "data" / "_downloads" / "dataset_phuonglam" for path in sorted(download_dir.glob("*")): if path.is_file(): local_files.append({"file": path.name, "bytes": path.stat().st_size}) write_text( OUT_ROOT / "data_audit" / "dataset_provenance.md", f""" # Dataset provenance audit ## Downloaded source The local datasets were downloaded from: `https://huggingface.co/datasets/minhy112/dataset_phuonglam/tree/main` Hugging Face API metadata captured during audit: ```json {json.dumps(hf_meta, indent=2, ensure_ascii=False)} ``` Local raw archives: ```json {json.dumps(local_files, indent=2, ensure_ascii=False)} ``` ## Current local extraction layout - `data/MVSA-Single/data/*.jpg|*.txt`, `data/MVSA-Single/labelResultAll.txt` - `data/MVSA-Multiple/data/*.jpg|*.txt`, `data/MVSA-Multiple/labelResultAll.txt` - `data/HFM/text/train.txt`, `val.txt`, `test.txt` - `data/HFM/train/image`, `data/HFM/val/image`, `data/HFM/test/image`, plus fallback `data/HFM/image` ## Provenance gaps that must be fixed in the paper 1. The Hugging Face dataset repository does not include a README, citation, or upstream-source manifest in the downloaded files. 2. HFM raw files are ZIP archives named `HFM-20260303T020235Z-1-001/002/003.zip`; the audit can document the local source but cannot prove the upstream/original dataset identity from metadata alone. 3. Current paper/config calls HFM binary `Hateful`/`Non-hateful`, but `src/hfm_pipeline.py` uses a 3-class default and sentiment label names (`positive`, `neutral`, `negative`). This must be resolved before reporting HFM results. 4. The HFM text files contain train/val/test files, but the audit cannot determine whether these are official splits without upstream documentation. State this explicitly unless a primary source is added. """, ) def write_architecture_reports() -> None: write_text( OUT_ROOT / "architecture" / "tensor_shapes.txt", """ Static tensor-shape audit after architecture correction. There are now three relevant model implementations: 0. Revised default implementation: `src/revised_ravel_model.py` - Selected by dataset pipeline config `architecture: token`. - Uses token-level co-attention. Expected revised token-level shapes: input_images: [B, 3, 224, 224] vision_encoder_output: [B, 197, 768] for CLIP ViT-B/16 at 224x224 visual_patch_tokens_raw: [B, 196, 768] projected_visual_tokens: [B, 196, 512] input_ids: [B, L] attention_mask: [B, L] text_encoder_output: [B, L, 768] projected_text_tokens: [B, L, 512] visual_to_text_attention_input: query [B, 196, 512], key/value [B, L, 512] text_to_visual_attention_input: query [B, L, 512], key/value [B, 196, 512] attention_v2t: [B, H, 196, L] attention_t2v: [B, H, L, 196] co_attention_output/fused: [B, 512] visual_logits: [B, C] text_logits: [B, C] pred_logits: [B, C] directional_disagreement: [B, C] = p_visual - p_text refined_logits/logits: [B, C] 1. `src/model.py` reference CLARAModel - Uses token-level encoder outputs: vision_outputs.last_hidden_state: [B, Nv, Dv] vision_projection(...): [B, Nv, 512] text_outputs.last_hidden_state: [B, Nt, Dt] text_projection(...): [B, Nt, 512] co-attention inputs: visual [B, Nv, 512], text [B, Nt, 512] - If `openai/clip-vit-base-patch16` at 224x224: Nv is expected to be 197 including CLS. - This implementation is not what the main train scripts import. 2. Legacy global baseline in the dataset pipelines: - `src/mvsa_single_pipeline.py` - `src/mvsa_multiple_pipeline.py` - `src/hfm_pipeline.py` - Selected by `architecture: legacy_global`. Legacy MVSA-Single / MVSA-Multiple pipeline forward: pixel_values: [B, 3, 224, 224] after CLIPProcessor vision_out.last_hidden_state: [B, Nv, Dv] vision_feat = mean(last_hidden_state, dim=1): [B, Dv] v_proj(vision_feat): [B, 512] vision_h = unsqueeze(1): [B, 1, 512] input_ids: [B, Nt] attention_mask: [B, Nt] text_out.last_hidden_state: [B, Nt, Dt] text_feat = masked mean over tokens: [B, Dt] t_proj(text_feat): [B, 512] text_h = unsqueeze(1): [B, 1, 512] visual_to_text_attention_input: query [B, 1, 512], key/value [B, 1, 512] text_to_visual_attention_input: query [B, 1, 512], key/value [B, 1, 512] attention weights if requested: [B, 1, 1] co_attention_output/fused: [B, 512] pooled_fused_output: not applicable; output is already global [B, 512] Legacy HFM pipeline forward: vision_out.last_hidden_state: [B, Nv, Dv] vision_feat = mean(last_hidden_state, dim=1): [B, Dv] text_out.last_hidden_state: [B, Nt, Dt] text_feat = masked mean over tokens: [B, Dt] CoAttentionStack: v = v_proj(v_feat).unsqueeze(1): [B, 1, 512] t = t_proj(t_feat).unsqueeze(1): [B, 1, 512] x = block(query=t, key_value=v): [B, 1, 512] fused = x.squeeze(1): [B, 512] Legacy conclusion: The legacy global baseline feeds `[B, 1, 512]` vs `[B, 1, 512]` into attention. Softmax over a single key is always 1, so legacy runs must be described as global-vector fusion, not region-phrase alignment. """, ) write_text( OUT_ROOT / "architecture" / "module_definitions.md", """ # Module and ablation definitions after architecture correction ## Revised token-level RAVEL (`architecture: token`) Implemented in `src/revised_ravel_model.py`. - Token-level co-attention input: - visual patch tokens: `[B, Nv, D]`, default `Nv=196` for CLIP ViT-B/16 - text token embeddings: `[B, Nt, D]` - `visual_head` consumes a visual-only global representation. - `text_head` consumes a text-only masked-mean representation. - directional disagreement is exactly `visual_probs - text_probs`. - final prediction uses DDCR: `refinement([h_co; disagreement])`. - `w/o Verification` / no-disagreement control: same refinement head with zero disagreement. - `w/o Feedback`: bypasses refinement and returns `pred_logits`. - `w/o Co-Attention`: fuses unimodal globals by normalized average and returns `pred`. - `Parameter-matched MLP`: uses `extra_mlp_control(h_co)` without disagreement. ## Legacy MVSA pipeline (`architecture: legacy_global`) 1. CLIP `last_hidden_state` is mean-pooled to one vision vector. 2. DeBERTa `last_hidden_state` is masked-mean-pooled to one text vector. 3. `CoAttentionFusion` projects both global vectors and applies multi-head attention after `unsqueeze(1)`. 4. `pred` head maps fused representation to `pred_logits`. 5. `veri` head maps the same fused representation to `verify_logits`. 6. The refinement signal is `softmax(verify_logits/tau) - softmax(pred_logits/tau)`. 7. In MVSA forward, this signal is detached. 8. `feed([fused; signal])` then `final` produces final logits. ## Legacy HFM pipeline (`architecture: legacy_global`) Same high-level idea, but `CoAttentionStack` only updates text query using vision key/value. It is not a symmetric bidirectional stack in the current HFM code. ## w/o Verification Current implementation: - keeps fused representation; - replaces disagreement/refinement signal with a zero vector; - still applies the `feed` + final/refined head. Interpretation: this is a no-disagreement-signal control, not a pure removal of all verification parameters. ## w/o Feedback Current implementation: - keeps fused representation; - bypasses `feed`/final refinement; - returns `pred(fused)`. Interpretation: removes the refinement head and the disagreement conditioning path. ## w/o Co-Attention Current implementation: - computes global `vision_feat` and `text_feat`; - applies `fusion.v_proj` and `fusion.t_proj`; - fuses by `layer_norm(0.5 * (v_hidden + t_hidden))`; - classifies with `pred`. ## Text-only / Vision-only Current implementation: - projects the single global modality vector through `fusion.t_proj` or `fusion.v_proj`; - classifies with `pred`. ## Important terminology correction The current experimental pipelines do not compute `p_v - p_t` from unimodal vision/text heads. They compute `p_verify - p_pred`, where both heads consume the same fused representation. Calling this a modality-level disagreement vector is not accurate for these pipelines. """, ) write_text( OUT_ROOT / "architecture" / "gradient_flow.md", """ # Gradient and loss audit Status after correction: - `architecture: token` uses non-detached directional disagreement from true unimodal heads. - revised loss includes `CE(visual_logits,y)` and `CE(text_logits,y)`. - legacy notes below document the frozen `architecture: legacy_global` baseline and the implementation issue found during audit. ## Legacy MVSA-Single / MVSA-Multiple forward ```python pred_logits = self.pred(fused) verify_logits = self.veri(fused) p_verify = softmax(verify_logits / tau).detach() p_pred = softmax(pred_logits / tau).detach() refined = self.feed(cat([fused, p_verify - p_pred])) logits = self.final(refined) ``` Consequences: - The disagreement signal is detached. - The final/refined loss does not backpropagate through `p_verify` or `p_pred`. - Gradients from final/refined loss still update `fused`, `feed`, `final`, and upstream encoders/fusion via `fused`. - `pred` and `veri` heads learn only if explicit losses are applied to `pred_logits` and `verify_logits`. ## MVSA default non-paper loss ```python loss_final = criterion(outputs["logits"], y) loss_pred = criterion(outputs["pred_logits"], y) loss_verify = criterion(outputs["verify_logits"], y) consistency = KL(p_pred || p_verify) + KL(p_verify || p_pred) full_loss = loss_final + pred_veri_weight*(loss_pred + loss_verify) + consistency_weight*consistency ``` This gives direct supervision to both auxiliary heads. ## MVSA-Multiple paper-loss mode ```python loss_primary = CE(pred_logits, y) loss_refined = CE(logits, y) full_loss = loss_primary + feedback_loss_weight * loss_refined ``` Because the signal is detached, `verify_logits` receives no gradient in this mode. If reviewer tables were generated with `--paper-loss-mode`, the verification head is effectively untrained unless loaded from a checkpoint trained with auxiliary verification loss. ## Legacy HFM forward ```python pred_logits, pred_probs = self.pred(fused) verify_logits, verify_probs = self.veri(fused) verification_signal = verify_probs - pred_probs refined = self.feed(fused, verification_signal) logits, probs = self.pred(refined) ``` Consequences: - The full HFM forward does not detach the signal. - Final loss can backpropagate through both `pred_probs` and `verify_probs`. - The same `pred` module is used before and after refinement. ## HFM default non-paper loss ```python loss_main = CE(logits, y) loss_verify = CE(verify_logits, y) loss_consistency = KL(logits || verify_logits) loss_contrastive = symmetric KL(pred_logits, verify_logits) if enabled loss = loss_main + loss_verify_weight*loss_verify + loss_consistency_weight*loss_consistency + contrastive_weight*loss_contrastive ``` ## HFM paper-loss mode ```python loss_primary = CE(pred_logits, y) loss_refined = CE(logits, y) loss = loss_primary + feedback_loss_weight * loss_refined ``` In HFM paper-loss mode, `verify_logits` can still receive gradient through the non-detached refinement signal, unlike MVSA. This means MVSA and HFM paper-loss modes are not computationally equivalent. ## Required paper correction Current "feedback" is prediction-head conditioning. It does not feed back into CLIP, DeBERTa, or attention weights. Safer names: - Directional disagreement-conditioned refinement - Disagreement-aware prediction refinement """, ) write_csv( OUT_ROOT / "architecture" / "loss_audit.csv", [ { "dataset_pipeline": "Revised token-level RAVEL", "mode": "default", "loss_primary": "lambda_primary * CE/Focal(pred_logits,y)", "loss_refined": "CE/Focal(logits,y)", "loss_visual": "lambda_unimodal * CE/Focal(visual_logits,y)", "loss_text": "lambda_unimodal * CE/Focal(text_logits,y)", "loss_pred_head": "yes via loss_primary", "loss_verify_head": "not used; true unimodal heads replace fused verification head", "loss_consistency": "none by default; use only as baseline/control", "label_smoothing": "dataset criterion config", "class_weighting": "dataset criterion/sampler config", "weighted_sampler": "dataset data-loader config", "signal_detached": "no", }, { "dataset_pipeline": "MVSA-Single", "mode": "default", "loss_primary": "CE/Focal(logits,y)", "loss_refined": "same as final logits", "loss_visual": "none", "loss_text": "none", "loss_pred_head": "pred_veri_weight * criterion(pred_logits,y)", "loss_verify_head": "pred_veri_weight * criterion(verify_logits,y)", "loss_consistency": "consistency_weight * symmetric KL(pred_probs,verify_probs)", "label_smoothing": "cfg label_smoothing, default 0.10", "class_weighting": "ExtremeFocalLoss class weights unless loss_type=ce without ce_use_class_weights", "weighted_sampler": "optional, default enabled in train script unless --disable-weighted-sampler", "signal_detached": "yes", }, { "dataset_pipeline": "MVSA-Multiple", "mode": "default non-paper", "loss_primary": "CE/Focal(logits,y)", "loss_refined": "same as final logits", "loss_visual": "none", "loss_text": "none", "loss_pred_head": "pred_veri_weight * criterion(pred_logits,y)", "loss_verify_head": "pred_veri_weight * criterion(verify_logits,y)", "loss_consistency": "consistency_weight * symmetric KL(pred_probs,verify_probs)", "label_smoothing": "cfg label_smoothing, default 0.02", "class_weighting": "ExtremeFocalLoss class weights", "weighted_sampler": "optional, default enabled", "signal_detached": "yes", }, { "dataset_pipeline": "MVSA-Multiple", "mode": "paper_loss_mode", "loss_primary": "CE(pred_logits,y)", "loss_refined": "feedback_loss_weight * CE(logits,y)", "loss_visual": "none", "loss_text": "none", "loss_pred_head": "yes via loss_primary", "loss_verify_head": "none; signal detached", "loss_consistency": "none", "label_smoothing": "cfg label_smoothing", "class_weighting": "none in CE", "weighted_sampler": "still controlled by data loader config", "signal_detached": "yes", }, { "dataset_pipeline": "HFM", "mode": "default non-paper", "loss_primary": "CE(logits,y)", "loss_refined": "same as final logits", "loss_visual": "none", "loss_text": "none", "loss_pred_head": "only through final path and contrastive if enabled", "loss_verify_head": "loss_verify_weight * CE(verify_logits,y)", "loss_consistency": "loss_consistency_weight * KL(logits,verify_logits)", "label_smoothing": "cfg label_smoothing, default 0.05", "class_weighting": "weighted sampler, not CE weights", "weighted_sampler": "optional, default enabled", "signal_detached": "no in full forward", }, ], [ "dataset_pipeline", "mode", "loss_primary", "loss_refined", "loss_visual", "loss_text", "loss_pred_head", "loss_verify_head", "loss_consistency", "label_smoothing", "class_weighting", "weighted_sampler", "signal_detached", ], ) write_text( OUT_ROOT / "architecture" / "parameter_breakdown.csv", "component,total_parameters,trainable_parameters,trainable_percentage\n" "NOT_GENERATED,current environment has no torch/transformers/peft; run scripts/probe_runtime_architecture.py after installing requirements,,\n", ) write_text( OUT_ROOT / "architecture" / "revised_architecture_spec.md", """ # Revised RAVEL architecture specification The revised implementation is in `src/revised_ravel_model.py` and is selected by `architecture: token`. ## Encoders - Vision backbone: CLIP ViT hidden states before pooling. - Default vision model id: `openai/clip-vit-base-patch16`. - Visual co-attention input excludes CLS by default: `visual_tokens_raw = vision_hidden[:, 1:, :]`. - Text backbone: DeBERTa last hidden states, not pooled output. ## Expected runtime shapes With 224x224 inputs and CLIP ViT-B/16: ```text input_images: [B, 3, 224, 224] vision_encoder_output: [B, 197, 768] visual_patch_tokens_raw: [B, 196, 768] projected_visual_tokens: [B, 196, 512] input_ids: [B, L] attention_mask: [B, L] text_encoder_output: [B, L, 768] projected_text_tokens: [B, L, 512] attention_v2t: [B, H, 196, L] attention_t2v: [B, H, L, 196] co_attention_output/fused: [B, 512] visual_logits: [B, C] text_logits: [B, C] pred_logits: [B, C] disagreement: [B, C] = softmax(visual_logits/disagreement_tau) - softmax(text_logits/disagreement_tau) logits/refined_logits: [B, C] ``` ## Heads and loss `visual_logits` and `text_logits` come from modality-specific representations before fusion. The default revised loss is: ```text L = CE(refined,y) + lambda_primary * CE(primary,y) + lambda_unimodal * (CE(visual,y) + CE(text,y)) ``` Defaults: - `lambda_primary = 0.5` - `lambda_unimodal = 0.25` - `disagreement_tau = 1.0` - `text_unfreeze_mode = freeze_all` for revised parameter-efficient runs ## Legacy baseline The old global-vector implementation is preserved as `LegacyGlobalCLARAModel` and selected by `architecture: legacy_global`. In reports/paper text this should be named `RAVEL-Global` or `global-token fusion variant`. """, ) write_csv( OUT_ROOT / "architecture" / "attention_statistics.csv", [], [ "dataset", "seed", "sample_id", "layer", "direction", "head", "attention_mean", "attention_std", "attention_entropy", "max_attention", "padding_attention_mass", ], ) write_csv( OUT_ROOT / "architecture" / "gradient_probe.csv", [], ["component", "parameter_name", "requires_grad", "gradient_present", "gradient_norm"], ) write_csv( OUT_ROOT / "architecture" / "modality_dependency_tests.csv", [], [ "test_case", "visual_posterior_change", "text_posterior_change", "expected_behavior", "passed", ], ) def write_empty_result_templates() -> None: main_fields = [ "dataset", "method", "configuration", "seed", "accuracy", "macro_precision", "macro_recall", "macro_f1", "weighted_f1", "auroc", "raw_ece", "ts_ece", "adaptive_ece", "nll", "brier", "aurc", "risk_at_80_coverage", "risk_at_90_coverage", "total_params", "trainable_params", "best_epoch", "training_time_seconds", "peak_training_vram_mb", ] for name in [ "main_results.csv", "component_ablation.csv", "disagreement_results.csv", "calibration_results.csv", "conflict_bins.csv", "stress_test_results.csv", "third_dataset_results.csv", "lvlm_results.csv", "efficiency_results.csv", "hyperparameter_results.csv", "significance_tests.csv", ]: write_csv(OUT_ROOT / "aggregate_results" / name, [], main_fields if name == "main_results.csv" else ["status", "note"]) write_csv( OUT_ROOT / "pilot" / "pilot_metrics.csv", [], ["dataset", "architecture", "seed", "epoch", "loss", "accuracy", "macro_f1", "raw_ece", "nll"], ) write_text( OUT_ROOT / "pilot" / "pilot_tensor_shapes.txt", "NOT_GENERATED: run a correctness pilot after installing torch/transformers/peft.\n", ) write_csv( OUT_ROOT / "pilot" / "pilot_training_curves.csv", [], ["dataset", "architecture", "seed", "epoch", "train_loss", "val_f1", "val_raw_ece"], ) write_text( OUT_ROOT / "pilot" / "pilot_failures.md", "# Pilot failures\n\nNo pilot run has been executed yet.\n", ) def write_readme( mvsa_single: List[MVSAAuditSample], mvsa_multiple: List[MVSAAuditSample], hfm_samples: List[HFMAuditSample], ) -> None: write_text( OUT_ROOT / "README.md", f""" # RAVEL revision results package This package currently contains Priority-0 audit/correction outputs only. No large experiment grid has been run yet. ## Completed - Dataset download/extraction audit. - MVSA-Single manifest and default split manifest. - MVSA-Multiple manifest and default split manifest. - HFM manifest filtered by images found by current loader logic. - HFM duplicate/leakage indicators. - Static architecture, tensor-shape, ablation-definition, loss, and gradient-flow audit. - Revised token-level architecture spec and runtime-probe templates. - HFM binary label mapping and HFM-DeLeak split manifest. ## Current status before large experiments 1. Revised training defaults now select `architecture: token`, which uses CLIP patch tokens and DeBERTa token embeddings for co-attention. 2. The old global-vector implementation is preserved as `architecture: legacy_global` / RAVEL-Global baseline. 3. Revised disagreement is computed from true unimodal heads: `p_visual - p_text`. 4. HFM is now binary in code (`0=Non-hateful`, `1=Hateful`) and has a generated HFM-DeLeak split manifest. 5. MVSA-Multiple has one image file per sample in this dataset. It should not be described as multi-image posts unless code/data are changed. 6. Runtime `named_parameters()`, actual tensor-shape, attention, and gradient reports still require installing `torch`, `transformers`, and `peft`. ## Current loaded sample counts - MVSA-Single: {len(mvsa_single)} - MVSA-Multiple: {len(mvsa_multiple)} - HFM samples with images found: {len(hfm_samples)} ## Next command after installing dependencies ```bash python scripts/probe_runtime_architecture.py --pipeline mvsa_multiple --output-dir ravel_revision_results/architecture python scripts/probe_runtime_architecture.py --pipeline hfm --output-dir ravel_revision_results/architecture ``` """, ) def main() -> None: ensure_dirs() hf_meta = fetch_hf_metadata() mvsa_single = load_mvsa_single(PROJECT_ROOT / "data" / "MVSA-Single") mvsa_multiple = load_mvsa_multiple(PROJECT_ROOT / "data" / "MVSA-Multiple", paper_exact_counts=True) hfm_samples, hfm_raw_counts, hfm_found_counts = load_hfm(PROJECT_ROOT / "data" / "HFM") single_split = split_mvsa(mvsa_single, seed=42) multiple_split = split_mvsa(mvsa_multiple, seed=42) write_environment_files() write_provenance(hf_meta) write_mvsa_manifest("split_manifest_mvsa_single.csv", single_split) write_mvsa_manifest("split_manifest_mvsa_multiple.csv", multiple_split) write_hfm_manifest(hfm_samples) write_csv( OUT_ROOT / "data_audit" / "dataset_statistics.csv", dataset_statistics_rows(mvsa_single, mvsa_multiple, hfm_samples, hfm_raw_counts, hfm_found_counts), [ "dataset", "loaded_samples", "raw_text_rows", "train_samples_with_images", "val_samples_with_images", "test_samples_with_images", "positive_or_label0", "neutral_or_label1", "negative_or_label2", "mean_images_per_sample", "median_images_per_sample", "max_images_per_sample", "samples_with_gt_1_image", "aggregation_in_current_code", ], ) write_multi_image_table(mvsa_single, mvsa_multiple, hfm_samples) write_hfm_duplicate_reports(hfm_samples) write_hfm_deleaked_split(hfm_samples, seed=42) write_architecture_reports() write_empty_result_templates() write_readme(mvsa_single, mvsa_multiple, hfm_samples) print(f"Wrote Priority-0 audit package to {OUT_ROOT}") print(f"MVSA-Single samples: {len(mvsa_single)}") print(f"MVSA-Multiple samples after paper exact counts: {len(mvsa_multiple)}") print(f"HFM loaded samples with images: {len(hfm_samples)}") if __name__ == "__main__": main()