Spaces:
Sleeping
Sleeping
File size: 3,091 Bytes
44b3467 7330a0e 44b3467 7330a0e 44b3467 7330a0e 44b3467 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | """Evaluate the WLASL Transformer + VLM reranking experiment.
Example:
python scripts/evaluate_hybrid_vlm.py \
--manifest data/vlm_eval_wlasl25_cnn/wlasl25_cnn_hybrid_eval.jsonl \
--output-dir results/vlm_eval
After filling the generated CSV's `vlm_prediction` column:
python scripts/evaluate_hybrid_vlm.py \
--manifest data/vlm_eval_wlasl25_cnn/wlasl25_cnn_hybrid_eval.jsonl \
--predictions results/vlm_eval/vlm_review_template.csv \
--output-dir results/vlm_eval
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[1]
SRC_ROOT = PROJECT_ROOT / "src"
if str(SRC_ROOT) not in sys.path:
sys.path.insert(0, str(SRC_ROOT))
from bridgelink_asl.hybrid_eval import (
compute_hybrid_metrics,
load_jsonl,
merge_predictions,
write_jsonl,
write_review_csv,
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Generate and score Transformer + VLM hybrid reranking metrics."
)
parser.add_argument(
"--manifest",
required=True,
type=Path,
help="Path to wlasl25_hybrid_eval.jsonl from the Colab notebook.",
)
parser.add_argument(
"--output-dir",
default=Path("results/vlm_eval"),
type=Path,
help="Directory for metrics, merged JSONL, and review CSV outputs.",
)
parser.add_argument(
"--predictions",
type=Path,
help=(
"Optional CSV or JSONL containing video_id plus vlm_prediction. "
"Use the generated review CSV after filling the vlm_prediction column."
),
)
return parser.parse_args()
def main() -> None:
args = parse_args()
rows = load_jsonl(args.manifest)
if args.predictions:
rows = merge_predictions(rows, args.predictions)
args.output_dir.mkdir(parents=True, exist_ok=True)
review_csv = args.output_dir / "vlm_review_template.csv"
merged_jsonl = args.output_dir / "vlm_hybrid_results.jsonl"
metrics_json = args.output_dir / "vlm_hybrid_metrics.json"
write_review_csv(rows, review_csv)
write_jsonl(rows, merged_jsonl)
metrics = compute_hybrid_metrics(rows)
with metrics_json.open("w", encoding="utf-8") as handle:
json.dump(metrics, handle, indent=2)
print(f"Samples: {metrics['num_samples']}")
print(f"Classes: {metrics['num_classes']}")
print(f"Candidate model: {metrics['candidate_model']}")
print(f"Candidate top-1 accuracy: {metrics['candidate_top1_accuracy']:.3f}")
print(f"Candidate top-5 coverage: {metrics['candidate_top5_coverage']:.3f}")
if metrics["vlm_rerank_accuracy"] is None:
print("VLM rerank accuracy: not scored yet")
print(f"Fill the vlm_prediction column in: {review_csv}")
else:
print(f"VLM rerank accuracy: {metrics['vlm_rerank_accuracy']:.3f}")
print(f"Wrote metrics: {metrics_json}")
print(f"Wrote merged rows: {merged_jsonl}")
if __name__ == "__main__":
main()
|