| |
| """Priority-0 audit for the RAVEL/CLARA revision. |
| |
| This script intentionally avoids torch/transformers so it can run before the |
| training environment is installed. It audits: |
| |
| - local dataset layout and split manifests; |
| - HFM provenance from the Hugging Face dataset repository; |
| - duplicate/leakage indicators for HFM; |
| - static architecture and loss/gradient-flow issues visible from code. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import ast |
| import csv |
| import hashlib |
| import json |
| import os |
| import platform |
| import random |
| import re |
| import statistics |
| import subprocess |
| import sys |
| import textwrap |
| import urllib.request |
| from collections import Counter, defaultdict |
| from dataclasses import dataclass |
| from pathlib import Path |
| from typing import Any, Dict, Iterable, List, Optional, Tuple |
|
|
|
|
| PROJECT_ROOT = Path(__file__).resolve().parents[1] |
| OUT_ROOT = PROJECT_ROOT / "ravel_revision_results" |
|
|
| LABELS = {"positive", "neutral", "negative"} |
| LABEL_NAME_TO_ID = {"positive": 0, "neutral": 1, "negative": 2} |
|
|
|
|
| @dataclass |
| class MVSAAuditSample: |
| dataset: str |
| sample_id: str |
| image_path: str |
| text_path: str |
| label: str |
| text_labels: List[str] |
| image_labels: List[str] |
| text_majority: str |
| image_majority: str |
|
|
|
|
| @dataclass |
| class HFMAuditSample: |
| sample_id: str |
| split: str |
| text: str |
| label: str |
| image_path: str |
| raw_record_len: int |
|
|
|
|
| def ensure_dirs() -> None: |
| for rel in [ |
| "environment", |
| "data_audit", |
| "architecture", |
| "configs/main", |
| "configs/ablation", |
| "configs/disagreement", |
| "configs/calibration", |
| "configs/stress_test", |
| "configs/hyperparameters", |
| "configs/lvlm", |
| "aggregate_results", |
| "predictions/mvsa", |
| "predictions/hfm", |
| "training_logs", |
| "attention_maps", |
| "error_analysis", |
| "figures", |
| "pilot", |
| ]: |
| (OUT_ROOT / rel).mkdir(parents=True, exist_ok=True) |
|
|
|
|
| def write_text(path: Path, content: str) -> None: |
| path.parent.mkdir(parents=True, exist_ok=True) |
| path.write_text(textwrap.dedent(content).strip() + "\n", encoding="utf-8") |
|
|
|
|
| def write_csv(path: Path, rows: Iterable[Dict[str, Any]], fieldnames: List[str]) -> None: |
| path.parent.mkdir(parents=True, exist_ok=True) |
| with path.open("w", newline="", encoding="utf-8") as f: |
| writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") |
| writer.writeheader() |
| for row in rows: |
| writer.writerow(row) |
|
|
|
|
| def run_text(cmd: List[str], cwd: Path = PROJECT_ROOT) -> str: |
| try: |
| proc = subprocess.run( |
| cmd, |
| cwd=str(cwd), |
| text=True, |
| capture_output=True, |
| check=False, |
| ) |
| out = (proc.stdout or "") + (proc.stderr or "") |
| return out.strip() |
| except Exception as exc: |
| return f"ERROR running {' '.join(cmd)}: {exc}" |
|
|
|
|
| def fetch_hf_metadata() -> Dict[str, Any]: |
| url = "https://huggingface.co/api/datasets/minhy112/dataset_phuonglam" |
| try: |
| with urllib.request.urlopen(url, timeout=20) as resp: |
| return json.loads(resp.read().decode("utf-8")) |
| except Exception as exc: |
| return {"error": str(exc), "url": url} |
|
|
|
|
| def parse_pair(value: str) -> Optional[Tuple[str, str]]: |
| parts = [token.strip().lower() for token in value.split(",")] |
| if len(parts) != 2: |
| return None |
| if parts[0] not in LABELS or parts[1] not in LABELS: |
| return None |
| return parts[0], parts[1] |
|
|
|
|
| def majority(labels: List[str]) -> str: |
| return Counter(labels).most_common(1)[0][0] |
|
|
|
|
| def fuse_modal_labels(text_label: str, image_label: str) -> Optional[str]: |
| if {text_label, image_label} == {"positive", "negative"}: |
| return None |
| if text_label == "positive" or image_label == "positive": |
| return "positive" |
| if text_label == "negative" or image_label == "negative": |
| return "negative" |
| return "neutral" |
|
|
|
|
| def find_image(data_dir: Path, sample_id: str) -> Optional[Path]: |
| for ext in [".jpg", ".JPG", ".jpeg", ".JPEG", ".png", ".PNG", ".bmp", ".BMP"]: |
| path = data_dir / f"{sample_id}{ext}" |
| if path.exists(): |
| return path |
| return None |
|
|
|
|
| def load_mvsa_single(root: Path) -> List[MVSAAuditSample]: |
| data_dir = root / "data" |
| label_file = root / "labelResultAll.txt" |
| lines = label_file.read_text(encoding="utf-8", errors="ignore").splitlines() |
| samples: List[MVSAAuditSample] = [] |
| for line in lines[1:]: |
| parts = line.strip().split("\t") |
| if len(parts) < 2: |
| continue |
| sample_id = parts[0].strip() |
| pair = parse_pair(parts[1]) |
| if pair is None: |
| continue |
| t1, i1 = pair |
| label = fuse_modal_labels(t1, i1) |
| if label is None: |
| continue |
| text_path = data_dir / f"{sample_id}.txt" |
| image_path = find_image(data_dir, sample_id) |
| if not text_path.exists() or image_path is None: |
| continue |
| samples.append( |
| MVSAAuditSample( |
| dataset="MVSA-Single", |
| sample_id=sample_id, |
| image_path=str(image_path), |
| text_path=str(text_path), |
| label=label, |
| text_labels=[t1], |
| image_labels=[i1], |
| text_majority=t1, |
| image_majority=i1, |
| ) |
| ) |
| return samples |
|
|
|
|
| def load_mvsa_multiple(root: Path, paper_exact_counts: bool = True) -> List[MVSAAuditSample]: |
| data_dir = root / "data" |
| label_file = root / "labelResultAll.txt" |
| lines = label_file.read_text(encoding="utf-8", errors="ignore").splitlines() |
| samples: List[MVSAAuditSample] = [] |
| for line in lines[1:]: |
| parts = line.strip().split("\t") |
| if len(parts) < 4: |
| continue |
| sample_id = parts[0].strip() |
| parsed = [parse_pair(parts[idx]) for idx in (1, 2, 3)] |
| if any(item is None for item in parsed): |
| continue |
| pairs = [item for item in parsed if item is not None] |
| text_labels = [pair[0] for pair in pairs] |
| image_labels = [pair[1] for pair in pairs] |
| if len(set(text_labels)) == 3 or len(set(image_labels)) == 3: |
| continue |
| text_majority = majority(text_labels) |
| image_majority = majority(image_labels) |
| label = fuse_modal_labels(text_majority, image_majority) |
| if label is None: |
| continue |
| text_path = data_dir / f"{sample_id}.txt" |
| image_path = find_image(data_dir, sample_id) |
| if not text_path.exists() or image_path is None: |
| continue |
| samples.append( |
| MVSAAuditSample( |
| dataset="MVSA-Multiple", |
| sample_id=sample_id, |
| image_path=str(image_path), |
| text_path=str(text_path), |
| label=label, |
| text_labels=text_labels, |
| image_labels=image_labels, |
| text_majority=text_majority, |
| image_majority=image_majority, |
| ) |
| ) |
|
|
| if paper_exact_counts: |
| targets = {"positive": 11318, "neutral": 4408, "negative": 1298} |
| kept: List[MVSAAuditSample] = [] |
| by_label: Dict[str, List[MVSAAuditSample]] = defaultdict(list) |
| for sample in samples: |
| by_label[sample.label].append(sample) |
| for label, target in targets.items(): |
| bucket = sorted(by_label[label], key=lambda item: int(item.sample_id)) |
| kept.extend(bucket[: min(target, len(bucket))]) |
| return kept |
| return samples |
|
|
|
|
| def split_mvsa( |
| samples: List[MVSAAuditSample], |
| train_ratio: float = 0.8, |
| val_ratio: float = 0.1, |
| seed: int = 42, |
| ) -> Dict[str, List[MVSAAuditSample]]: |
| random.seed(seed) |
| groups: Dict[str, List[MVSAAuditSample]] = {"positive": [], "neutral": [], "negative": []} |
| for sample in samples: |
| groups[sample.label].append(sample) |
|
|
| split_map = {"train": [], "val": [], "test": []} |
| for group in groups.values(): |
| random.shuffle(group) |
| n = len(group) |
| n_train = int(n * train_ratio) |
| n_val = int(n * val_ratio) |
| split_map["train"].extend(group[:n_train]) |
| split_map["val"].extend(group[n_train : n_train + n_val]) |
| split_map["test"].extend(group[n_train + n_val :]) |
|
|
| random.shuffle(split_map["train"]) |
| random.shuffle(split_map["val"]) |
| random.shuffle(split_map["test"]) |
| return split_map |
|
|
|
|
| def normalize_text(text: str) -> str: |
| return re.sub(r"\s+", " ", text.strip().lower()) |
|
|
|
|
| def hfm_image_dirs(root: Path, split: str) -> List[Path]: |
| dirs = [root / split / "image"] |
| if split == "val": |
| dirs.append(root / "valid" / "image") |
| dirs.append(root / "image") |
| return dirs |
|
|
|
|
| def find_hfm_image(root: Path, split: str, sample_id: str) -> Optional[Path]: |
| for image_dir in hfm_image_dirs(root, split): |
| if not image_dir.exists(): |
| continue |
| for ext in [".jpg", ".JPG", ".jpeg", ".JPEG", ".png", ".PNG", ".bmp"]: |
| path = image_dir / f"{sample_id}{ext}" |
| if path.exists(): |
| return path |
| return None |
|
|
|
|
| def load_hfm(root: Path) -> Tuple[List[HFMAuditSample], Dict[str, int], Dict[str, int]]: |
| text_dir = root / "text" |
| loaded: List[HFMAuditSample] = [] |
| raw_counts: Dict[str, int] = {} |
| found_counts: Dict[str, int] = {} |
| for filename, split in [("train.txt", "train"), ("val.txt", "val"), ("test.txt", "test")]: |
| raw_counts[split] = 0 |
| found_counts[split] = 0 |
| file_path = text_dir / filename |
| if not file_path.exists(): |
| continue |
| for line in file_path.read_text(encoding="utf-8", errors="ignore").splitlines(): |
| line = line.strip() |
| if not line: |
| continue |
| try: |
| record = ast.literal_eval(line) |
| except Exception: |
| continue |
| if not isinstance(record, (list, tuple)) or len(record) < 3: |
| continue |
| raw_counts[split] += 1 |
| sample_id = str(record[0]).strip() |
| text = str(record[1]).strip() |
| label = str(record[2]).strip() |
| image_path = find_hfm_image(root, split, sample_id) |
| if image_path is None: |
| continue |
| found_counts[split] += 1 |
| loaded.append( |
| HFMAuditSample( |
| sample_id=sample_id, |
| split=split, |
| text=text, |
| label=label, |
| image_path=str(image_path), |
| raw_record_len=len(record), |
| ) |
| ) |
| return loaded, raw_counts, found_counts |
|
|
|
|
| def sha256_file(path: Path) -> str: |
| h = hashlib.sha256() |
| with path.open("rb") as f: |
| while True: |
| chunk = f.read(1024 * 1024) |
| if not chunk: |
| break |
| h.update(chunk) |
| return h.hexdigest() |
|
|
|
|
| def summarize_counter(counter: Counter) -> str: |
| return json.dumps(dict(sorted(counter.items())), ensure_ascii=False) |
|
|
|
|
| def write_mvsa_manifest(filename: str, split_map: Dict[str, List[MVSAAuditSample]]) -> None: |
| rows: List[Dict[str, Any]] = [] |
| for split, samples in split_map.items(): |
| for sample in samples: |
| rows.append( |
| { |
| "dataset": sample.dataset, |
| "split": split, |
| "sample_id": sample.sample_id, |
| "label": sample.label, |
| "label_id": LABEL_NAME_TO_ID[sample.label], |
| "image_path": sample.image_path, |
| "text_path": sample.text_path, |
| "number_of_images": 1, |
| "text_labels": "|".join(sample.text_labels), |
| "image_labels": "|".join(sample.image_labels), |
| "text_majority": sample.text_majority, |
| "image_majority": sample.image_majority, |
| } |
| ) |
| write_csv( |
| OUT_ROOT / "data_audit" / filename, |
| rows, |
| [ |
| "dataset", |
| "split", |
| "sample_id", |
| "label", |
| "label_id", |
| "image_path", |
| "text_path", |
| "number_of_images", |
| "text_labels", |
| "image_labels", |
| "text_majority", |
| "image_majority", |
| ], |
| ) |
|
|
|
|
| def write_hfm_manifest(samples: List[HFMAuditSample]) -> None: |
| rows = [] |
| for sample in samples: |
| label_id = _hfm_binary_label(sample.label) |
| rows.append( |
| { |
| "dataset": "HFM", |
| "split": sample.split, |
| "sample_id": sample.sample_id, |
| "raw_label": sample.label, |
| "label_id": "" if label_id is None else label_id, |
| "label_name": "" if label_id is None else _hfm_label_name(label_id), |
| "label_mapping_status": "binary_hfm_mapping" if label_id is not None else "invalid_label", |
| "image_path": sample.image_path, |
| "text": sample.text, |
| "text_norm": normalize_text(sample.text), |
| "number_of_images": 1, |
| "raw_record_len": sample.raw_record_len, |
| } |
| ) |
| write_csv( |
| OUT_ROOT / "data_audit" / "split_manifest_hfm.csv", |
| rows, |
| [ |
| "dataset", |
| "split", |
| "sample_id", |
| "raw_label", |
| "label_id", |
| "label_name", |
| "label_mapping_status", |
| "image_path", |
| "text", |
| "text_norm", |
| "number_of_images", |
| "raw_record_len", |
| ], |
| ) |
|
|
|
|
| def dataset_statistics_rows( |
| mvsa_single: List[MVSAAuditSample], |
| mvsa_multiple: List[MVSAAuditSample], |
| hfm_samples: List[HFMAuditSample], |
| hfm_raw_counts: Dict[str, int], |
| hfm_found_counts: Dict[str, int], |
| ) -> List[Dict[str, Any]]: |
| rows: List[Dict[str, Any]] = [] |
| for dataset, samples in [ |
| ("MVSA-Single", mvsa_single), |
| ("MVSA-Multiple", mvsa_multiple), |
| ]: |
| counts = Counter(sample.label for sample in samples) |
| rows.append( |
| { |
| "dataset": dataset, |
| "loaded_samples": len(samples), |
| "raw_text_rows": "", |
| "train_samples_with_images": "", |
| "val_samples_with_images": "", |
| "test_samples_with_images": "", |
| "positive_or_label0": counts.get("positive", 0), |
| "neutral_or_label1": counts.get("neutral", 0), |
| "negative_or_label2": counts.get("negative", 0), |
| "mean_images_per_sample": "1.0000", |
| "median_images_per_sample": "1", |
| "max_images_per_sample": "1", |
| "samples_with_gt_1_image": "0", |
| "aggregation_in_current_code": "single image; CLIP tokens mean-pooled before fusion", |
| } |
| ) |
|
|
| hfm_counts = Counter(sample.label for sample in hfm_samples) |
| rows.append( |
| { |
| "dataset": "HFM", |
| "loaded_samples": len(hfm_samples), |
| "raw_text_rows": sum(hfm_raw_counts.values()), |
| "train_samples_with_images": hfm_found_counts.get("train", 0), |
| "val_samples_with_images": hfm_found_counts.get("val", 0), |
| "test_samples_with_images": hfm_found_counts.get("test", 0), |
| "positive_or_label0": hfm_counts.get("0", 0), |
| "neutral_or_label1": hfm_counts.get("1", 0), |
| "negative_or_label2": hfm_counts.get("2", 0), |
| "mean_images_per_sample": "1.0000", |
| "median_images_per_sample": "1", |
| "max_images_per_sample": "1", |
| "samples_with_gt_1_image": "0", |
| "aggregation_in_current_code": "single image; CLIP tokens mean-pooled before fusion", |
| } |
| ) |
| return rows |
|
|
|
|
| def write_multi_image_table( |
| mvsa_single: List[MVSAAuditSample], |
| mvsa_multiple: List[MVSAAuditSample], |
| hfm_samples: List[HFMAuditSample], |
| ) -> None: |
| rows = [ |
| { |
| "Dataset": "MVSA-Single", |
| "Samples": len(mvsa_single), |
| "Mean images/sample": "1.0000", |
| "Median": 1, |
| "Max": 1, |
| "Samples with >1 image": 0, |
| "Aggregation": "none; one image file per sample; model mean-pools CLIP tokens", |
| }, |
| { |
| "Dataset": "MVSA-Multiple", |
| "Samples": len(mvsa_multiple), |
| "Mean images/sample": "1.0000", |
| "Median": 1, |
| "Max": 1, |
| "Samples with >1 image": 0, |
| "Aggregation": "none; one image file per sample; MVSA-Multiple refers annotations, not multi-image posts", |
| }, |
| { |
| "Dataset": "HFM", |
| "Samples": len(hfm_samples), |
| "Mean images/sample": "1.0000", |
| "Median": 1, |
| "Max": 1, |
| "Samples with >1 image": 0, |
| "Aggregation": "none; one image file per loaded sample; model mean-pools CLIP tokens", |
| }, |
| ] |
| write_csv( |
| OUT_ROOT / "data_audit" / "multi_image_handling.csv", |
| rows, |
| [ |
| "Dataset", |
| "Samples", |
| "Mean images/sample", |
| "Median", |
| "Max", |
| "Samples with >1 image", |
| "Aggregation", |
| ], |
| ) |
|
|
|
|
| def write_hfm_duplicate_reports(samples: List[HFMAuditSample]) -> None: |
| duplicate_rows: List[Dict[str, Any]] = [] |
| leakage_rows: List[Dict[str, Any]] = [] |
|
|
| def add_duplicate(kind: str, key: str, grouped: List[HFMAuditSample]) -> None: |
| splits = sorted({sample.split for sample in grouped}) |
| if len(grouped) <= 1: |
| return |
| duplicate_rows.append( |
| { |
| "dataset": "HFM", |
| "duplicate_type": kind, |
| "key": key, |
| "total_occurrences": len(grouped), |
| "splits": "|".join(splits), |
| "cross_split": int(len(splits) > 1), |
| "sample_ids": "|".join(sample.sample_id for sample in grouped[:50]), |
| "labels": "|".join(sample.label for sample in grouped[:50]), |
| "paths": "|".join(sample.image_path for sample in grouped[:20]), |
| } |
| ) |
|
|
| by_id: Dict[str, List[HFMAuditSample]] = defaultdict(list) |
| by_text: Dict[str, List[HFMAuditSample]] = defaultdict(list) |
| for sample in samples: |
| by_id[sample.sample_id].append(sample) |
| by_text[normalize_text(sample.text)].append(sample) |
|
|
| for key, grouped in by_id.items(): |
| add_duplicate("image_id", key, grouped) |
| for key, grouped in by_text.items(): |
| add_duplicate("exact_normalized_text", hashlib.sha1(key.encode("utf-8")).hexdigest(), grouped) |
|
|
| hash_cache: Dict[str, str] = {} |
| by_hash: Dict[str, List[HFMAuditSample]] = defaultdict(list) |
| for sample in samples: |
| path = sample.image_path |
| if path not in hash_cache: |
| hash_cache[path] = sha256_file(Path(path)) |
| by_hash[hash_cache[path]].append(sample) |
| for key, grouped in by_hash.items(): |
| add_duplicate("image_sha256", key, grouped) |
|
|
| write_csv( |
| OUT_ROOT / "data_audit" / "duplicate_report.csv", |
| duplicate_rows, |
| [ |
| "dataset", |
| "duplicate_type", |
| "key", |
| "total_occurrences", |
| "splits", |
| "cross_split", |
| "sample_ids", |
| "labels", |
| "paths", |
| ], |
| ) |
|
|
| for kind in ["image_id", "exact_normalized_text", "image_sha256"]: |
| rows = [row for row in duplicate_rows if row["duplicate_type"] == kind] |
| cross = [row for row in rows if str(row["cross_split"]) == "1"] |
| severity = "high" if kind in {"image_id", "image_sha256"} and cross else ("medium" if cross else "none") |
| leakage_rows.append( |
| { |
| "dataset": "HFM", |
| "check": kind, |
| "finding": f"{len(rows)} duplicate keys; {len(cross)} cross-split duplicate keys", |
| "severity": severity, |
| "details": "See duplicate_report.csv", |
| } |
| ) |
|
|
| train_ids = {sample.sample_id for sample in samples if sample.split == "train"} |
| val_ids = {sample.sample_id for sample in samples if sample.split == "val"} |
| test_ids = {sample.sample_id for sample in samples if sample.split == "test"} |
| leakage_rows.append( |
| { |
| "dataset": "HFM", |
| "check": "split_id_overlap", |
| "finding": json.dumps( |
| { |
| "train_val": len(train_ids & val_ids), |
| "train_test": len(train_ids & test_ids), |
| "val_test": len(val_ids & test_ids), |
| } |
| ), |
| "severity": "high" if (train_ids & val_ids or train_ids & test_ids or val_ids & test_ids) else "none", |
| "details": "Overlap is computed on raw image/text sample IDs after filtering to samples with images.", |
| } |
| ) |
|
|
| write_csv( |
| OUT_ROOT / "data_audit" / "leakage_report.csv", |
| leakage_rows, |
| ["dataset", "check", "finding", "severity", "details"], |
| ) |
|
|
|
|
| class UnionFind: |
| def __init__(self, n: int): |
| self.parent = list(range(n)) |
| self.rank = [0] * n |
|
|
| def find(self, x: int) -> int: |
| while self.parent[x] != x: |
| self.parent[x] = self.parent[self.parent[x]] |
| x = self.parent[x] |
| return x |
|
|
| def union(self, a: int, b: int) -> None: |
| ra = self.find(a) |
| rb = self.find(b) |
| if ra == rb: |
| return |
| if self.rank[ra] < self.rank[rb]: |
| ra, rb = rb, ra |
| self.parent[rb] = ra |
| if self.rank[ra] == self.rank[rb]: |
| self.rank[ra] += 1 |
|
|
|
|
| def _hfm_binary_label(raw_label: str) -> Optional[int]: |
| value = str(raw_label).strip().lower() |
| if value in {"0", "non-hateful", "non_hateful", "non hateful", "not hateful", "benign"}: |
| return 0 |
| if value in {"1", "hateful", "hate"}: |
| return 1 |
| return None |
|
|
|
|
| def _hfm_label_name(label_id: int) -> str: |
| return "Non-hateful" if int(label_id) == 0 else "Hateful" |
|
|
|
|
| def _cross_split_key_count(key_to_samples: Dict[str, List[HFMAuditSample]]) -> int: |
| return sum(1 for grouped in key_to_samples.values() if len({sample.split for sample in grouped}) > 1) |
|
|
|
|
| def write_hfm_deleaked_split(samples: List[HFMAuditSample], seed: int = 42) -> None: |
| """Build duplicate-connected HFM components and a group-stratified split.""" |
| n = len(samples) |
| uf = UnionFind(n) |
| text_to_indices: Dict[str, List[int]] = defaultdict(list) |
| hash_to_indices: Dict[str, List[int]] = defaultdict(list) |
| image_hashes: Dict[int, str] = {} |
|
|
| for idx, sample in enumerate(samples): |
| text_to_indices[normalize_text(sample.text)].append(idx) |
| image_hash = sha256_file(Path(sample.image_path)) |
| image_hashes[idx] = image_hash |
| hash_to_indices[image_hash].append(idx) |
|
|
| for grouped in list(text_to_indices.values()) + list(hash_to_indices.values()): |
| if len(grouped) <= 1: |
| continue |
| first = grouped[0] |
| for other in grouped[1:]: |
| uf.union(first, other) |
|
|
| components_by_root: Dict[int, List[int]] = defaultdict(list) |
| for idx in range(n): |
| components_by_root[uf.find(idx)].append(idx) |
|
|
| components = list(components_by_root.values()) |
|
|
| total_label_counts = Counter() |
| for sample in samples: |
| label_id = _hfm_binary_label(sample.label) |
| if label_id is not None: |
| total_label_counts[label_id] += 1 |
|
|
| ratios = {"train": 0.8, "val": 0.1, "test": 0.1} |
| desired_total = {split: ratios[split] * len(samples) for split in ratios} |
| desired_label = { |
| split: {label: ratios[split] * count for label, count in total_label_counts.items()} |
| for split in ratios |
| } |
| component_label_counts: List[Counter] = [] |
| for indices in components: |
| comp_counts = Counter() |
| for idx in indices: |
| label_id = _hfm_binary_label(samples[idx].label) |
| if label_id is not None: |
| comp_counts[label_id] += 1 |
| component_label_counts.append(comp_counts) |
|
|
| def objective(sizes: Counter, counts: Dict[str, Counter]) -> float: |
| score = 0.0 |
| for split in ["train", "val", "test"]: |
| score += 4.0 * ( |
| (sizes[split] - desired_total[split]) / max(1.0, desired_total[split]) |
| ) ** 2 |
| for label in sorted(total_label_counts): |
| score += 10.0 * ( |
| (counts[split][label] - desired_label[split][label]) |
| / max(1.0, desired_label[split][label]) |
| ) ** 2 |
| return score |
|
|
| best_component_split: Dict[int, str] = {} |
| best_split_sizes: Counter = Counter() |
| best_score = float("inf") |
| rng = random.Random(seed) |
| attempts = 96 |
| component_ids = list(range(len(components))) |
|
|
| for _attempt in range(attempts): |
| ordered = sorted( |
| component_ids, |
| key=lambda cid: (-len(components[cid]), rng.random()), |
| ) |
| split_counts = {split: Counter() for split in ratios} |
| split_sizes = Counter() |
| candidate_split: Dict[int, str] = {} |
|
|
| for component_id in ordered: |
| indices = components[component_id] |
| comp_counts = component_label_counts[component_id] |
| chosen_split = "train" |
| chosen_score = float("inf") |
| for split in ["train", "val", "test"]: |
| projected_sizes = Counter(split_sizes) |
| projected_counts = {name: Counter(values) for name, values in split_counts.items()} |
| projected_sizes[split] += len(indices) |
| projected_counts[split].update(comp_counts) |
| score = objective(projected_sizes, projected_counts) |
| if score < chosen_score: |
| chosen_score = score |
| chosen_split = split |
| candidate_split[component_id] = chosen_split |
| split_sizes[chosen_split] += len(indices) |
| split_counts[chosen_split].update(comp_counts) |
|
|
| score = objective(split_sizes, split_counts) |
| if score < best_score: |
| best_score = score |
| best_component_split = candidate_split |
| best_split_sizes = Counter(split_sizes) |
|
|
| component_split = best_component_split |
| split_sizes = best_split_sizes |
|
|
| component_rows: List[Dict[str, Any]] = [] |
| manifest_rows: List[Dict[str, Any]] = [] |
| for component_id, indices in enumerate(components): |
| deleaked_split = component_split[component_id] |
| component_splits = sorted({samples[idx].split for idx in indices}) |
| component_labels = Counter() |
| for idx in indices: |
| label_id = _hfm_binary_label(samples[idx].label) |
| if label_id is not None: |
| component_labels[label_id] += 1 |
| for idx in indices: |
| sample = samples[idx] |
| label_id = _hfm_binary_label(sample.label) |
| if label_id is None: |
| continue |
| row = { |
| "dataset": "HFM-DeLeak", |
| "duplicate_component_id": component_id, |
| "component_size": len(indices), |
| "component_original_splits": "|".join(component_splits), |
| "component_label_counts": json.dumps(dict(component_labels), sort_keys=True), |
| "original_split": sample.split, |
| "split": deleaked_split, |
| "sample_id": sample.sample_id, |
| "label_id": label_id, |
| "label_name": _hfm_label_name(label_id), |
| "raw_label": sample.label, |
| "image_sha256": image_hashes[idx], |
| "image_path": sample.image_path, |
| "text_norm": normalize_text(sample.text), |
| "text": sample.text, |
| } |
| component_rows.append(row) |
| manifest_rows.append(row) |
|
|
| fieldnames = [ |
| "dataset", |
| "duplicate_component_id", |
| "component_size", |
| "component_original_splits", |
| "component_label_counts", |
| "original_split", |
| "split", |
| "sample_id", |
| "label_id", |
| "label_name", |
| "raw_label", |
| "image_sha256", |
| "image_path", |
| "text_norm", |
| "text", |
| ] |
| write_csv(OUT_ROOT / "data_audit" / "hfm_duplicate_components.csv", component_rows, fieldnames) |
| write_csv(OUT_ROOT / "data_audit" / "hfm_split_manifest_deleaked.csv", manifest_rows, fieldnames) |
|
|
| def stats_rows(rows: List[Dict[str, Any]], split_key: str) -> List[Dict[str, Any]]: |
| out: List[Dict[str, Any]] = [] |
| for split in ["train", "val", "test"]: |
| split_rows = [row for row in rows if row[split_key] == split] |
| counts = Counter(int(row["label_id"]) for row in split_rows) |
| out.append( |
| { |
| "protocol": "HFM-DeLeak" if split_key == "split" else "Existing", |
| "split": split, |
| "samples": len(split_rows), |
| "non_hateful": counts.get(0, 0), |
| "hateful": counts.get(1, 0), |
| } |
| ) |
| return out |
|
|
| write_csv( |
| OUT_ROOT / "data_audit" / "hfm_existing_split_statistics.csv", |
| stats_rows(component_rows, "original_split"), |
| ["protocol", "split", "samples", "non_hateful", "hateful"], |
| ) |
| write_csv( |
| OUT_ROOT / "data_audit" / "hfm_deleaked_split_statistics.csv", |
| stats_rows(component_rows, "split"), |
| ["protocol", "split", "samples", "non_hateful", "hateful"], |
| ) |
|
|
| existing_text_cross = _cross_split_key_count( |
| {key: [samples[idx] for idx in indices] for key, indices in text_to_indices.items()} |
| ) |
| existing_image_cross = _cross_split_key_count( |
| {key: [samples[idx] for idx in indices] for key, indices in hash_to_indices.items()} |
| ) |
| comparison_rows = [ |
| { |
| "Protocol": "Existing", |
| "Train": sum(1 for sample in samples if sample.split == "train"), |
| "Val": sum(1 for sample in samples if sample.split == "val"), |
| "Test": sum(1 for sample in samples if sample.split == "test"), |
| "Cross-split text duplicates": existing_text_cross, |
| "Cross-split image duplicates": existing_image_cross, |
| }, |
| { |
| "Protocol": "HFM-DeLeak", |
| "Train": split_sizes["train"], |
| "Val": split_sizes["val"], |
| "Test": split_sizes["test"], |
| "Cross-split text duplicates": 0, |
| "Cross-split image duplicates": 0, |
| }, |
| ] |
| write_csv( |
| OUT_ROOT / "data_audit" / "hfm_split_protocol_comparison.csv", |
| comparison_rows, |
| [ |
| "Protocol", |
| "Train", |
| "Val", |
| "Test", |
| "Cross-split text duplicates", |
| "Cross-split image duplicates", |
| ], |
| ) |
|
|
| write_csv( |
| OUT_ROOT / "data_audit" / "hfm_label_mapping.csv", |
| [ |
| {"label_id": 0, "label_name": "Non-hateful", "positive_class_for_auroc": 0}, |
| {"label_id": 1, "label_name": "Hateful", "positive_class_for_auroc": 1}, |
| ], |
| ["label_id", "label_name", "positive_class_for_auroc"], |
| ) |
|
|
| write_text( |
| OUT_ROOT / "data_audit" / "hfm_provenance.md", |
| f""" |
| # HFM provenance and split audit |
| |
| ## Local source |
| |
| Local data was downloaded from: |
| |
| `https://huggingface.co/datasets/minhy112/dataset_phuonglam/tree/main` |
| |
| Local HFM extraction root: |
| |
| `data/HFM` |
| |
| Text files used by the current loader: |
| |
| - `data/HFM/text/train.txt` |
| - `data/HFM/text/val.txt` |
| - `data/HFM/text/test.txt` |
| |
| ## Current sample count |
| |
| Records with image files found by the loader: {len(samples)} |
| |
| Existing split counts: |
| |
| ```json |
| {json.dumps({split: sum(1 for sample in samples if sample.split == split) for split in ["train", "val", "test"]}, indent=2)} |
| ``` |
| |
| ## Label mapping |
| |
| The revised HFM pipeline uses a binary output space: |
| |
| - `0`: Non-hateful |
| - `1`: Hateful |
| |
| The AUROC positive class is `1` (`Hateful`). |
| |
| ## Provenance limitation |
| |
| The downloaded Hugging Face repository does not include a README, |
| citation file, official-split statement, augmentation manifest, or |
| upstream dataset hash manifest. Unless a primary upstream source is |
| added later, the paper should describe this corpus as the local |
| HFM-derived corpus from the cited repository, not as proof that the |
| 24,635-record split is the original official Hateful Memes Challenge |
| corpus. |
| |
| ## Duplicate/leakage handling |
| |
| A duplicate graph was built over samples. Edges connect samples when |
| they have the same exact normalized text or the same image SHA256 hash. |
| Connected components were split as groups to produce `HFM-DeLeak`. |
| |
| Output files: |
| |
| - `hfm_duplicate_components.csv` |
| - `hfm_split_manifest_deleaked.csv` |
| - `hfm_existing_split_statistics.csv` |
| - `hfm_deleaked_split_statistics.csv` |
| - `hfm_split_protocol_comparison.csv` |
| """, |
| ) |
|
|
|
|
| def write_environment_files() -> None: |
| write_text( |
| OUT_ROOT / "environment" / "hardware.txt", |
| f""" |
| platform: {platform.platform()} |
| python: {sys.version.replace(os.linesep, " ")} |
| processor: {platform.processor()} |
| |
| uname: |
| {run_text(["uname", "-a"])} |
| |
| lscpu: |
| {run_text(["lscpu"])} |
| |
| nvidia-smi: |
| {run_text(["nvidia-smi"])} |
| """, |
| ) |
| write_text(OUT_ROOT / "environment" / "pip_freeze.txt", run_text([sys.executable, "-m", "pip", "freeze"])) |
| write_text(OUT_ROOT / "environment" / "git_commit.txt", run_text(["git", "rev-parse", "HEAD"])) |
| write_text( |
| OUT_ROOT / "environment" / "environment.yml", |
| """ |
| name: ravel-revision-audit |
| note: Current execution environment lacks torch/transformers/peft/Pillow/sklearn. |
| install_project_requirements: pip install -r requirements.txt |
| runtime_probe_after_install: python scripts/probe_runtime_architecture.py --pipeline mvsa_multiple --output-dir ravel_revision_results/architecture |
| """, |
| ) |
|
|
|
|
| def write_provenance(hf_meta: Dict[str, Any]) -> None: |
| local_files = [] |
| download_dir = PROJECT_ROOT / "data" / "_downloads" / "dataset_phuonglam" |
| for path in sorted(download_dir.glob("*")): |
| if path.is_file(): |
| local_files.append({"file": path.name, "bytes": path.stat().st_size}) |
|
|
| write_text( |
| OUT_ROOT / "data_audit" / "dataset_provenance.md", |
| f""" |
| # Dataset provenance audit |
| |
| ## Downloaded source |
| |
| The local datasets were downloaded from: |
| |
| `https://huggingface.co/datasets/minhy112/dataset_phuonglam/tree/main` |
| |
| Hugging Face API metadata captured during audit: |
| |
| ```json |
| {json.dumps(hf_meta, indent=2, ensure_ascii=False)} |
| ``` |
| |
| Local raw archives: |
| |
| ```json |
| {json.dumps(local_files, indent=2, ensure_ascii=False)} |
| ``` |
| |
| ## Current local extraction layout |
| |
| - `data/MVSA-Single/data/*.jpg|*.txt`, `data/MVSA-Single/labelResultAll.txt` |
| - `data/MVSA-Multiple/data/*.jpg|*.txt`, `data/MVSA-Multiple/labelResultAll.txt` |
| - `data/HFM/text/train.txt`, `val.txt`, `test.txt` |
| - `data/HFM/train/image`, `data/HFM/val/image`, `data/HFM/test/image`, plus fallback `data/HFM/image` |
| |
| ## Provenance gaps that must be fixed in the paper |
| |
| 1. The Hugging Face dataset repository does not include a README, citation, or upstream-source manifest in the downloaded files. |
| 2. HFM raw files are ZIP archives named `HFM-20260303T020235Z-1-001/002/003.zip`; the audit can document the local source but cannot prove the upstream/original dataset identity from metadata alone. |
| 3. Current paper/config calls HFM binary `Hateful`/`Non-hateful`, but `src/hfm_pipeline.py` uses a 3-class default and sentiment label names (`positive`, `neutral`, `negative`). This must be resolved before reporting HFM results. |
| 4. The HFM text files contain train/val/test files, but the audit cannot determine whether these are official splits without upstream documentation. State this explicitly unless a primary source is added. |
| """, |
| ) |
|
|
|
|
| def write_architecture_reports() -> None: |
| write_text( |
| OUT_ROOT / "architecture" / "tensor_shapes.txt", |
| """ |
| Static tensor-shape audit after architecture correction. |
| |
| There are now three relevant model implementations: |
| |
| 0. Revised default implementation: `src/revised_ravel_model.py` |
| - Selected by dataset pipeline config `architecture: token`. |
| - Uses token-level co-attention. |
| |
| Expected revised token-level shapes: |
| |
| input_images: [B, 3, 224, 224] |
| vision_encoder_output: [B, 197, 768] for CLIP ViT-B/16 at 224x224 |
| visual_patch_tokens_raw: [B, 196, 768] |
| projected_visual_tokens: [B, 196, 512] |
| |
| input_ids: [B, L] |
| attention_mask: [B, L] |
| text_encoder_output: [B, L, 768] |
| projected_text_tokens: [B, L, 512] |
| |
| visual_to_text_attention_input: query [B, 196, 512], key/value [B, L, 512] |
| text_to_visual_attention_input: query [B, L, 512], key/value [B, 196, 512] |
| attention_v2t: [B, H, 196, L] |
| attention_t2v: [B, H, L, 196] |
| co_attention_output/fused: [B, 512] |
| |
| visual_logits: [B, C] |
| text_logits: [B, C] |
| pred_logits: [B, C] |
| directional_disagreement: [B, C] = p_visual - p_text |
| refined_logits/logits: [B, C] |
| |
| 1. `src/model.py` reference CLARAModel |
| - Uses token-level encoder outputs: |
| vision_outputs.last_hidden_state: [B, Nv, Dv] |
| vision_projection(...): [B, Nv, 512] |
| text_outputs.last_hidden_state: [B, Nt, Dt] |
| text_projection(...): [B, Nt, 512] |
| co-attention inputs: visual [B, Nv, 512], text [B, Nt, 512] |
| - If `openai/clip-vit-base-patch16` at 224x224: Nv is expected to be 197 including CLS. |
| - This implementation is not what the main train scripts import. |
| |
| 2. Legacy global baseline in the dataset pipelines: |
| - `src/mvsa_single_pipeline.py` |
| - `src/mvsa_multiple_pipeline.py` |
| - `src/hfm_pipeline.py` |
| - Selected by `architecture: legacy_global`. |
| |
| Legacy MVSA-Single / MVSA-Multiple pipeline forward: |
| |
| pixel_values: [B, 3, 224, 224] after CLIPProcessor |
| vision_out.last_hidden_state: [B, Nv, Dv] |
| vision_feat = mean(last_hidden_state, dim=1): [B, Dv] |
| v_proj(vision_feat): [B, 512] |
| vision_h = unsqueeze(1): [B, 1, 512] |
| |
| input_ids: [B, Nt] |
| attention_mask: [B, Nt] |
| text_out.last_hidden_state: [B, Nt, Dt] |
| text_feat = masked mean over tokens: [B, Dt] |
| t_proj(text_feat): [B, 512] |
| text_h = unsqueeze(1): [B, 1, 512] |
| |
| visual_to_text_attention_input: query [B, 1, 512], key/value [B, 1, 512] |
| text_to_visual_attention_input: query [B, 1, 512], key/value [B, 1, 512] |
| attention weights if requested: [B, 1, 1] |
| co_attention_output/fused: [B, 512] |
| pooled_fused_output: not applicable; output is already global [B, 512] |
| |
| Legacy HFM pipeline forward: |
| |
| vision_out.last_hidden_state: [B, Nv, Dv] |
| vision_feat = mean(last_hidden_state, dim=1): [B, Dv] |
| text_out.last_hidden_state: [B, Nt, Dt] |
| text_feat = masked mean over tokens: [B, Dt] |
| CoAttentionStack: |
| v = v_proj(v_feat).unsqueeze(1): [B, 1, 512] |
| t = t_proj(t_feat).unsqueeze(1): [B, 1, 512] |
| x = block(query=t, key_value=v): [B, 1, 512] |
| fused = x.squeeze(1): [B, 512] |
| |
| Legacy conclusion: |
| |
| The legacy global baseline feeds `[B, 1, 512]` vs `[B, 1, 512]` into attention. Softmax over a single key is always 1, so legacy runs must be described as global-vector fusion, not region-phrase alignment. |
| """, |
| ) |
|
|
| write_text( |
| OUT_ROOT / "architecture" / "module_definitions.md", |
| """ |
| # Module and ablation definitions after architecture correction |
| |
| ## Revised token-level RAVEL (`architecture: token`) |
| |
| Implemented in `src/revised_ravel_model.py`. |
| |
| - Token-level co-attention input: |
| - visual patch tokens: `[B, Nv, D]`, default `Nv=196` for CLIP ViT-B/16 |
| - text token embeddings: `[B, Nt, D]` |
| - `visual_head` consumes a visual-only global representation. |
| - `text_head` consumes a text-only masked-mean representation. |
| - directional disagreement is exactly `visual_probs - text_probs`. |
| - final prediction uses DDCR: `refinement([h_co; disagreement])`. |
| - `w/o Verification` / no-disagreement control: same refinement head with zero disagreement. |
| - `w/o Feedback`: bypasses refinement and returns `pred_logits`. |
| - `w/o Co-Attention`: fuses unimodal globals by normalized average and returns `pred`. |
| - `Parameter-matched MLP`: uses `extra_mlp_control(h_co)` without disagreement. |
| |
| ## Legacy MVSA pipeline (`architecture: legacy_global`) |
| |
| 1. CLIP `last_hidden_state` is mean-pooled to one vision vector. |
| 2. DeBERTa `last_hidden_state` is masked-mean-pooled to one text vector. |
| 3. `CoAttentionFusion` projects both global vectors and applies multi-head attention after `unsqueeze(1)`. |
| 4. `pred` head maps fused representation to `pred_logits`. |
| 5. `veri` head maps the same fused representation to `verify_logits`. |
| 6. The refinement signal is `softmax(verify_logits/tau) - softmax(pred_logits/tau)`. |
| 7. In MVSA forward, this signal is detached. |
| 8. `feed([fused; signal])` then `final` produces final logits. |
| |
| ## Legacy HFM pipeline (`architecture: legacy_global`) |
| |
| Same high-level idea, but `CoAttentionStack` only updates text query using vision key/value. It is not a symmetric bidirectional stack in the current HFM code. |
| |
| ## w/o Verification |
| |
| Current implementation: |
| |
| - keeps fused representation; |
| - replaces disagreement/refinement signal with a zero vector; |
| - still applies the `feed` + final/refined head. |
| |
| Interpretation: this is a no-disagreement-signal control, not a pure removal of all verification parameters. |
| |
| ## w/o Feedback |
| |
| Current implementation: |
| |
| - keeps fused representation; |
| - bypasses `feed`/final refinement; |
| - returns `pred(fused)`. |
| |
| Interpretation: removes the refinement head and the disagreement conditioning path. |
| |
| ## w/o Co-Attention |
| |
| Current implementation: |
| |
| - computes global `vision_feat` and `text_feat`; |
| - applies `fusion.v_proj` and `fusion.t_proj`; |
| - fuses by `layer_norm(0.5 * (v_hidden + t_hidden))`; |
| - classifies with `pred`. |
| |
| ## Text-only / Vision-only |
| |
| Current implementation: |
| |
| - projects the single global modality vector through `fusion.t_proj` or `fusion.v_proj`; |
| - classifies with `pred`. |
| |
| ## Important terminology correction |
| |
| The current experimental pipelines do not compute `p_v - p_t` from unimodal vision/text heads. They compute `p_verify - p_pred`, where both heads consume the same fused representation. Calling this a modality-level disagreement vector is not accurate for these pipelines. |
| """, |
| ) |
|
|
| write_text( |
| OUT_ROOT / "architecture" / "gradient_flow.md", |
| """ |
| # Gradient and loss audit |
| |
| Status after correction: |
| |
| - `architecture: token` uses non-detached directional disagreement from true unimodal heads. |
| - revised loss includes `CE(visual_logits,y)` and `CE(text_logits,y)`. |
| - legacy notes below document the frozen `architecture: legacy_global` baseline and the implementation issue found during audit. |
| |
| ## Legacy MVSA-Single / MVSA-Multiple forward |
| |
| ```python |
| pred_logits = self.pred(fused) |
| verify_logits = self.veri(fused) |
| p_verify = softmax(verify_logits / tau).detach() |
| p_pred = softmax(pred_logits / tau).detach() |
| refined = self.feed(cat([fused, p_verify - p_pred])) |
| logits = self.final(refined) |
| ``` |
| |
| Consequences: |
| |
| - The disagreement signal is detached. |
| - The final/refined loss does not backpropagate through `p_verify` or `p_pred`. |
| - Gradients from final/refined loss still update `fused`, `feed`, `final`, and upstream encoders/fusion via `fused`. |
| - `pred` and `veri` heads learn only if explicit losses are applied to `pred_logits` and `verify_logits`. |
| |
| ## MVSA default non-paper loss |
| |
| ```python |
| loss_final = criterion(outputs["logits"], y) |
| loss_pred = criterion(outputs["pred_logits"], y) |
| loss_verify = criterion(outputs["verify_logits"], y) |
| consistency = KL(p_pred || p_verify) + KL(p_verify || p_pred) |
| full_loss = loss_final + pred_veri_weight*(loss_pred + loss_verify) + consistency_weight*consistency |
| ``` |
| |
| This gives direct supervision to both auxiliary heads. |
| |
| ## MVSA-Multiple paper-loss mode |
| |
| ```python |
| loss_primary = CE(pred_logits, y) |
| loss_refined = CE(logits, y) |
| full_loss = loss_primary + feedback_loss_weight * loss_refined |
| ``` |
| |
| Because the signal is detached, `verify_logits` receives no gradient in this mode. If reviewer tables were generated with `--paper-loss-mode`, the verification head is effectively untrained unless loaded from a checkpoint trained with auxiliary verification loss. |
| |
| ## Legacy HFM forward |
| |
| ```python |
| pred_logits, pred_probs = self.pred(fused) |
| verify_logits, verify_probs = self.veri(fused) |
| verification_signal = verify_probs - pred_probs |
| refined = self.feed(fused, verification_signal) |
| logits, probs = self.pred(refined) |
| ``` |
| |
| Consequences: |
| |
| - The full HFM forward does not detach the signal. |
| - Final loss can backpropagate through both `pred_probs` and `verify_probs`. |
| - The same `pred` module is used before and after refinement. |
| |
| ## HFM default non-paper loss |
| |
| ```python |
| loss_main = CE(logits, y) |
| loss_verify = CE(verify_logits, y) |
| loss_consistency = KL(logits || verify_logits) |
| loss_contrastive = symmetric KL(pred_logits, verify_logits) if enabled |
| loss = loss_main + loss_verify_weight*loss_verify + loss_consistency_weight*loss_consistency + contrastive_weight*loss_contrastive |
| ``` |
| |
| ## HFM paper-loss mode |
| |
| ```python |
| loss_primary = CE(pred_logits, y) |
| loss_refined = CE(logits, y) |
| loss = loss_primary + feedback_loss_weight * loss_refined |
| ``` |
| |
| In HFM paper-loss mode, `verify_logits` can still receive gradient through the non-detached refinement signal, unlike MVSA. This means MVSA and HFM paper-loss modes are not computationally equivalent. |
| |
| ## Required paper correction |
| |
| Current "feedback" is prediction-head conditioning. It does not feed back into CLIP, DeBERTa, or attention weights. Safer names: |
| |
| - Directional disagreement-conditioned refinement |
| - Disagreement-aware prediction refinement |
| """, |
| ) |
|
|
| write_csv( |
| OUT_ROOT / "architecture" / "loss_audit.csv", |
| [ |
| { |
| "dataset_pipeline": "Revised token-level RAVEL", |
| "mode": "default", |
| "loss_primary": "lambda_primary * CE/Focal(pred_logits,y)", |
| "loss_refined": "CE/Focal(logits,y)", |
| "loss_visual": "lambda_unimodal * CE/Focal(visual_logits,y)", |
| "loss_text": "lambda_unimodal * CE/Focal(text_logits,y)", |
| "loss_pred_head": "yes via loss_primary", |
| "loss_verify_head": "not used; true unimodal heads replace fused verification head", |
| "loss_consistency": "none by default; use only as baseline/control", |
| "label_smoothing": "dataset criterion config", |
| "class_weighting": "dataset criterion/sampler config", |
| "weighted_sampler": "dataset data-loader config", |
| "signal_detached": "no", |
| }, |
| { |
| "dataset_pipeline": "MVSA-Single", |
| "mode": "default", |
| "loss_primary": "CE/Focal(logits,y)", |
| "loss_refined": "same as final logits", |
| "loss_visual": "none", |
| "loss_text": "none", |
| "loss_pred_head": "pred_veri_weight * criterion(pred_logits,y)", |
| "loss_verify_head": "pred_veri_weight * criterion(verify_logits,y)", |
| "loss_consistency": "consistency_weight * symmetric KL(pred_probs,verify_probs)", |
| "label_smoothing": "cfg label_smoothing, default 0.10", |
| "class_weighting": "ExtremeFocalLoss class weights unless loss_type=ce without ce_use_class_weights", |
| "weighted_sampler": "optional, default enabled in train script unless --disable-weighted-sampler", |
| "signal_detached": "yes", |
| }, |
| { |
| "dataset_pipeline": "MVSA-Multiple", |
| "mode": "default non-paper", |
| "loss_primary": "CE/Focal(logits,y)", |
| "loss_refined": "same as final logits", |
| "loss_visual": "none", |
| "loss_text": "none", |
| "loss_pred_head": "pred_veri_weight * criterion(pred_logits,y)", |
| "loss_verify_head": "pred_veri_weight * criterion(verify_logits,y)", |
| "loss_consistency": "consistency_weight * symmetric KL(pred_probs,verify_probs)", |
| "label_smoothing": "cfg label_smoothing, default 0.02", |
| "class_weighting": "ExtremeFocalLoss class weights", |
| "weighted_sampler": "optional, default enabled", |
| "signal_detached": "yes", |
| }, |
| { |
| "dataset_pipeline": "MVSA-Multiple", |
| "mode": "paper_loss_mode", |
| "loss_primary": "CE(pred_logits,y)", |
| "loss_refined": "feedback_loss_weight * CE(logits,y)", |
| "loss_visual": "none", |
| "loss_text": "none", |
| "loss_pred_head": "yes via loss_primary", |
| "loss_verify_head": "none; signal detached", |
| "loss_consistency": "none", |
| "label_smoothing": "cfg label_smoothing", |
| "class_weighting": "none in CE", |
| "weighted_sampler": "still controlled by data loader config", |
| "signal_detached": "yes", |
| }, |
| { |
| "dataset_pipeline": "HFM", |
| "mode": "default non-paper", |
| "loss_primary": "CE(logits,y)", |
| "loss_refined": "same as final logits", |
| "loss_visual": "none", |
| "loss_text": "none", |
| "loss_pred_head": "only through final path and contrastive if enabled", |
| "loss_verify_head": "loss_verify_weight * CE(verify_logits,y)", |
| "loss_consistency": "loss_consistency_weight * KL(logits,verify_logits)", |
| "label_smoothing": "cfg label_smoothing, default 0.05", |
| "class_weighting": "weighted sampler, not CE weights", |
| "weighted_sampler": "optional, default enabled", |
| "signal_detached": "no in full forward", |
| }, |
| ], |
| [ |
| "dataset_pipeline", |
| "mode", |
| "loss_primary", |
| "loss_refined", |
| "loss_visual", |
| "loss_text", |
| "loss_pred_head", |
| "loss_verify_head", |
| "loss_consistency", |
| "label_smoothing", |
| "class_weighting", |
| "weighted_sampler", |
| "signal_detached", |
| ], |
| ) |
|
|
| write_text( |
| OUT_ROOT / "architecture" / "parameter_breakdown.csv", |
| "component,total_parameters,trainable_parameters,trainable_percentage\n" |
| "NOT_GENERATED,current environment has no torch/transformers/peft; run scripts/probe_runtime_architecture.py after installing requirements,,\n", |
| ) |
|
|
| write_text( |
| OUT_ROOT / "architecture" / "revised_architecture_spec.md", |
| """ |
| # Revised RAVEL architecture specification |
| |
| The revised implementation is in `src/revised_ravel_model.py` and is |
| selected by `architecture: token`. |
| |
| ## Encoders |
| |
| - Vision backbone: CLIP ViT hidden states before pooling. |
| - Default vision model id: `openai/clip-vit-base-patch16`. |
| - Visual co-attention input excludes CLS by default: |
| `visual_tokens_raw = vision_hidden[:, 1:, :]`. |
| - Text backbone: DeBERTa last hidden states, not pooled output. |
| |
| ## Expected runtime shapes |
| |
| With 224x224 inputs and CLIP ViT-B/16: |
| |
| ```text |
| input_images: [B, 3, 224, 224] |
| vision_encoder_output: [B, 197, 768] |
| visual_patch_tokens_raw: [B, 196, 768] |
| projected_visual_tokens: [B, 196, 512] |
| |
| input_ids: [B, L] |
| attention_mask: [B, L] |
| text_encoder_output: [B, L, 768] |
| projected_text_tokens: [B, L, 512] |
| |
| attention_v2t: [B, H, 196, L] |
| attention_t2v: [B, H, L, 196] |
| co_attention_output/fused: [B, 512] |
| |
| visual_logits: [B, C] |
| text_logits: [B, C] |
| pred_logits: [B, C] |
| disagreement: [B, C] = softmax(visual_logits/disagreement_tau) - softmax(text_logits/disagreement_tau) |
| logits/refined_logits: [B, C] |
| ``` |
| |
| ## Heads and loss |
| |
| `visual_logits` and `text_logits` come from modality-specific |
| representations before fusion. The default revised loss is: |
| |
| ```text |
| L = CE(refined,y) |
| + lambda_primary * CE(primary,y) |
| + lambda_unimodal * (CE(visual,y) + CE(text,y)) |
| ``` |
| |
| Defaults: |
| |
| - `lambda_primary = 0.5` |
| - `lambda_unimodal = 0.25` |
| - `disagreement_tau = 1.0` |
| - `text_unfreeze_mode = freeze_all` for revised parameter-efficient runs |
| |
| ## Legacy baseline |
| |
| The old global-vector implementation is preserved as |
| `LegacyGlobalCLARAModel` and selected by `architecture: legacy_global`. |
| In reports/paper text this should be named `RAVEL-Global` or |
| `global-token fusion variant`. |
| """, |
| ) |
|
|
| write_csv( |
| OUT_ROOT / "architecture" / "attention_statistics.csv", |
| [], |
| [ |
| "dataset", |
| "seed", |
| "sample_id", |
| "layer", |
| "direction", |
| "head", |
| "attention_mean", |
| "attention_std", |
| "attention_entropy", |
| "max_attention", |
| "padding_attention_mass", |
| ], |
| ) |
| write_csv( |
| OUT_ROOT / "architecture" / "gradient_probe.csv", |
| [], |
| ["component", "parameter_name", "requires_grad", "gradient_present", "gradient_norm"], |
| ) |
| write_csv( |
| OUT_ROOT / "architecture" / "modality_dependency_tests.csv", |
| [], |
| [ |
| "test_case", |
| "visual_posterior_change", |
| "text_posterior_change", |
| "expected_behavior", |
| "passed", |
| ], |
| ) |
|
|
|
|
| def write_empty_result_templates() -> None: |
| main_fields = [ |
| "dataset", |
| "method", |
| "configuration", |
| "seed", |
| "accuracy", |
| "macro_precision", |
| "macro_recall", |
| "macro_f1", |
| "weighted_f1", |
| "auroc", |
| "raw_ece", |
| "ts_ece", |
| "adaptive_ece", |
| "nll", |
| "brier", |
| "aurc", |
| "risk_at_80_coverage", |
| "risk_at_90_coverage", |
| "total_params", |
| "trainable_params", |
| "best_epoch", |
| "training_time_seconds", |
| "peak_training_vram_mb", |
| ] |
| for name in [ |
| "main_results.csv", |
| "component_ablation.csv", |
| "disagreement_results.csv", |
| "calibration_results.csv", |
| "conflict_bins.csv", |
| "stress_test_results.csv", |
| "third_dataset_results.csv", |
| "lvlm_results.csv", |
| "efficiency_results.csv", |
| "hyperparameter_results.csv", |
| "significance_tests.csv", |
| ]: |
| write_csv(OUT_ROOT / "aggregate_results" / name, [], main_fields if name == "main_results.csv" else ["status", "note"]) |
|
|
| write_csv( |
| OUT_ROOT / "pilot" / "pilot_metrics.csv", |
| [], |
| ["dataset", "architecture", "seed", "epoch", "loss", "accuracy", "macro_f1", "raw_ece", "nll"], |
| ) |
| write_text( |
| OUT_ROOT / "pilot" / "pilot_tensor_shapes.txt", |
| "NOT_GENERATED: run a correctness pilot after installing torch/transformers/peft.\n", |
| ) |
| write_csv( |
| OUT_ROOT / "pilot" / "pilot_training_curves.csv", |
| [], |
| ["dataset", "architecture", "seed", "epoch", "train_loss", "val_f1", "val_raw_ece"], |
| ) |
| write_text( |
| OUT_ROOT / "pilot" / "pilot_failures.md", |
| "# Pilot failures\n\nNo pilot run has been executed yet.\n", |
| ) |
|
|
|
|
| def write_readme( |
| mvsa_single: List[MVSAAuditSample], |
| mvsa_multiple: List[MVSAAuditSample], |
| hfm_samples: List[HFMAuditSample], |
| ) -> None: |
| write_text( |
| OUT_ROOT / "README.md", |
| f""" |
| # RAVEL revision results package |
| |
| This package currently contains Priority-0 audit/correction outputs |
| only. No large experiment grid has been run yet. |
| |
| ## Completed |
| |
| - Dataset download/extraction audit. |
| - MVSA-Single manifest and default split manifest. |
| - MVSA-Multiple manifest and default split manifest. |
| - HFM manifest filtered by images found by current loader logic. |
| - HFM duplicate/leakage indicators. |
| - Static architecture, tensor-shape, ablation-definition, loss, and gradient-flow audit. |
| - Revised token-level architecture spec and runtime-probe templates. |
| - HFM binary label mapping and HFM-DeLeak split manifest. |
| |
| ## Current status before large experiments |
| |
| 1. Revised training defaults now select `architecture: token`, which |
| uses CLIP patch tokens and DeBERTa token embeddings for co-attention. |
| 2. The old global-vector implementation is preserved as |
| `architecture: legacy_global` / RAVEL-Global baseline. |
| 3. Revised disagreement is computed from true unimodal heads: |
| `p_visual - p_text`. |
| 4. HFM is now binary in code (`0=Non-hateful`, `1=Hateful`) and has a |
| generated HFM-DeLeak split manifest. |
| 5. MVSA-Multiple has one image file per sample in this dataset. It |
| should not be described as multi-image posts unless code/data are changed. |
| 6. Runtime `named_parameters()`, actual tensor-shape, attention, and |
| gradient reports still require installing `torch`, `transformers`, and `peft`. |
| |
| ## Current loaded sample counts |
| |
| - MVSA-Single: {len(mvsa_single)} |
| - MVSA-Multiple: {len(mvsa_multiple)} |
| - HFM samples with images found: {len(hfm_samples)} |
| |
| ## Next command after installing dependencies |
| |
| ```bash |
| python scripts/probe_runtime_architecture.py --pipeline mvsa_multiple --output-dir ravel_revision_results/architecture |
| python scripts/probe_runtime_architecture.py --pipeline hfm --output-dir ravel_revision_results/architecture |
| ``` |
| """, |
| ) |
|
|
|
|
| def main() -> None: |
| ensure_dirs() |
|
|
| hf_meta = fetch_hf_metadata() |
| mvsa_single = load_mvsa_single(PROJECT_ROOT / "data" / "MVSA-Single") |
| mvsa_multiple = load_mvsa_multiple(PROJECT_ROOT / "data" / "MVSA-Multiple", paper_exact_counts=True) |
| hfm_samples, hfm_raw_counts, hfm_found_counts = load_hfm(PROJECT_ROOT / "data" / "HFM") |
|
|
| single_split = split_mvsa(mvsa_single, seed=42) |
| multiple_split = split_mvsa(mvsa_multiple, seed=42) |
|
|
| write_environment_files() |
| write_provenance(hf_meta) |
| write_mvsa_manifest("split_manifest_mvsa_single.csv", single_split) |
| write_mvsa_manifest("split_manifest_mvsa_multiple.csv", multiple_split) |
| write_hfm_manifest(hfm_samples) |
| write_csv( |
| OUT_ROOT / "data_audit" / "dataset_statistics.csv", |
| dataset_statistics_rows(mvsa_single, mvsa_multiple, hfm_samples, hfm_raw_counts, hfm_found_counts), |
| [ |
| "dataset", |
| "loaded_samples", |
| "raw_text_rows", |
| "train_samples_with_images", |
| "val_samples_with_images", |
| "test_samples_with_images", |
| "positive_or_label0", |
| "neutral_or_label1", |
| "negative_or_label2", |
| "mean_images_per_sample", |
| "median_images_per_sample", |
| "max_images_per_sample", |
| "samples_with_gt_1_image", |
| "aggregation_in_current_code", |
| ], |
| ) |
| write_multi_image_table(mvsa_single, mvsa_multiple, hfm_samples) |
| write_hfm_duplicate_reports(hfm_samples) |
| write_hfm_deleaked_split(hfm_samples, seed=42) |
| write_architecture_reports() |
| write_empty_result_templates() |
| write_readme(mvsa_single, mvsa_multiple, hfm_samples) |
|
|
| print(f"Wrote Priority-0 audit package to {OUT_ROOT}") |
| print(f"MVSA-Single samples: {len(mvsa_single)}") |
| print(f"MVSA-Multiple samples after paper exact counts: {len(mvsa_multiple)}") |
| print(f"HFM loaded samples with images: {len(hfm_samples)}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
