metadata/eval_suite_inventory.csv

suite_id,family,metric_group,source_file,source_datasets,target_comparisons,budget,sample_unit,comparison_image_keys,caption_records,surfaces,model_requests,atomic_questions,prompt_records,metric_cells,judge_or_encoder,notes
dataset_release_metadata,dataset_release,caption metadata records,dataset_release/caption_records_metadata.csv,"CC12M, Danbooru2023, DataComp, LAION-pop, PD12M","cc12m_four_caption_llava_url_bridge, cc12m_llavanext_paired, cc12m_pixelprose_paired, cc12m_qwen3vl8b_paired, danbooru2023_florence2_paired, datacomp_recap_llava15_paired_url, laion_pop_llama32_paired, pd12m_full_paired",mixed; explicit token_budget field per row,caption record with non-distributed image reference,39494,87976,18,0,0,0,0,,metadata-only release; images are not redistributed
cpu_text_cc12m_llavanext_paired,lexical_cpu,"length, surface concentration, lexical diversity, repetition, register artifact screen",eval_results/raw_summaries/cpu_text_metrics/fair_slices_1m_normalized_2026-04-24.tsv,CC12M,cc12m_llavanext_paired,"full and B in {16,32,64,128,256}",same-image paired caption rows,960394,1920788,2,0,0,0,2,regex lexical tokenizer,"CPU-only screening; @B uses lexical units, not model tokenizer tokens"
cpu_text_cc12m_pixelprose_paired,lexical_cpu,"length, surface concentration, lexical diversity, repetition, register artifact screen",eval_results/raw_summaries/cpu_text_metrics/fair_slices_1m_normalized_2026-04-24.tsv,CC12M,cc12m_pixelprose_paired,"full and B in {16,32,64,128,256}",same-image paired caption rows,729237,1458474,2,0,0,0,2,regex lexical tokenizer,"CPU-only screening; @B uses lexical units, not model tokenizer tokens"
cpu_text_cc12m_qwen3vl8b_paired,lexical_cpu,"length, surface concentration, lexical diversity, repetition, register artifact screen",eval_results/raw_summaries/cpu_text_metrics/fair_slices_1m_normalized_2026-04-24.tsv,CC12M,cc12m_qwen3vl8b_paired,"full and B in {16,32,64,128,256}",same-image paired caption rows,114621,229242,2,0,0,0,2,regex lexical tokenizer,"CPU-only screening; @B uses lexical units, not model tokenizer tokens"
cpu_text_danbooru2023_florence2_paired,lexical_cpu,"length, surface concentration, lexical diversity, repetition, register artifact screen",eval_results/raw_summaries/cpu_text_metrics/fair_slices_1m_normalized_2026-04-24.tsv,DANBOORU2023,danbooru2023_florence2_paired,"full and B in {16,32,64,128,256}",same-image paired caption rows,999993,1999986,2,0,0,0,2,regex lexical tokenizer,"CPU-only screening; @B uses lexical units, not model tokenizer tokens"
cpu_text_datacomp_recap_llava15_paired_url,lexical_cpu,"length, surface concentration, lexical diversity, repetition, register artifact screen",eval_results/raw_summaries/cpu_text_metrics/fair_slices_1m_normalized_2026-04-24.tsv,DATACOMP,datacomp_recap_llava15_paired_url,"full and B in {16,32,64,128,256}",same-image paired caption rows,1000000,2000000,2,0,0,0,2,regex lexical tokenizer,"CPU-only screening; @B uses lexical units, not model tokenizer tokens"
cpu_text_laion_pop_llama32_paired,lexical_cpu,"length, surface concentration, lexical diversity, repetition, register artifact screen",eval_results/raw_summaries/cpu_text_metrics/fair_slices_1m_normalized_2026-04-24.tsv,LAION,laion_pop_llama32_paired,"full and B in {16,32,64,128,256}",same-image paired caption rows,42231,84462,2,0,0,0,2,regex lexical tokenizer,"CPU-only screening; @B uses lexical units, not model tokenizer tokens"
cpu_text_pd12m_full_paired,lexical_cpu,"length, surface concentration, lexical diversity, repetition, register artifact screen",eval_results/raw_summaries/cpu_text_metrics/fair_slices_1m_normalized_2026-04-24.tsv,PD12M,pd12m_full_paired,"full and B in {16,32,64,128,256}",same-image paired caption rows,1000000,2000000,2,0,0,0,2,regex lexical tokenizer,"CPU-only screening; @B uses lexical units, not model tokenizer tokens"
prompt_ngram_support_250k_1mprompt_2026-04-24,lexical_prompt_support,hashed n-gram distribution support and distance,eval_results/raw_summaries/prompt_support/prompt_ngram_support_250k_1mprompt_2026-04-24.json,all paired slices in file,"cc12m_llavanext_paired, cc12m_pixelprose_paired, cc12m_qwen3vl8b_paired, danbooru2023_florence2_paired, datacomp_recap_llava15_paired_url, laion_pop_llama32_paired, pd12m_full_paired","64,128",caption pool vs declared prompt pool,,58126356,12,0,0,2007412,252,"hashed [1, 2, 3]-gram, 262144 buckets",caption_records counts metric-cell record instances because each budget/ngram/pool is measured separately
prompt_ngram_support_diffusiondb_official_250k_1mprompt_2026-04-24,lexical_prompt_support,hashed n-gram distribution support and distance,eval_results/raw_summaries/prompt_support/prompt_ngram_support_diffusiondb_official_250k_1mprompt_2026-04-24.json,all paired slices in file,"cc12m_llavanext_paired, cc12m_pixelprose_paired, cc12m_qwen3vl8b_paired, danbooru2023_florence2_paired, datacomp_recap_llava15_paired_url, laion_pop_llama32_paired, pd12m_full_paired","64,128",caption pool vs declared prompt pool,,19375452,12,0,0,1000000,84,"hashed [1, 2, 3]-gram, 262144 buckets",caption_records counts metric-cell record instances because each budget/ngram/pool is measured separately
prompt_ngram_support_expanded_250k_1mprompt_2026-04-24,lexical_prompt_support,hashed n-gram distribution support and distance,eval_results/raw_summaries/prompt_support/prompt_ngram_support_expanded_250k_1mprompt_2026-04-24.json,all paired slices in file,"cc12m_llavanext_paired, cc12m_pixelprose_paired, cc12m_qwen3vl8b_paired, danbooru2023_florence2_paired, datacomp_recap_llava15_paired_url, laion_pop_llama32_paired, pd12m_full_paired","64,128",caption pool vs declared prompt pool,,155003616,12,0,0,2049431,672,"hashed [1, 2, 3]-gram, 262144 buckets",caption_records counts metric-cell record instances because each budget/ngram/pool is measured separately
prompt_support_bootstrap_b64_n2,lexical_prompt_support,paired block-bootstrap support deltas,eval_results/raw_summaries/prompt_support/prompt_support_bootstrap_b64_n2_250k_2026-04-24.json,all paired slices in file,"cc12m_llavanext_paired, cc12m_pixelprose_paired, cc12m_qwen3vl8b_paired, danbooru2023_florence2_paired, datacomp_recap_llava15_paired_url, laion_pop_llama32_paired, pd12m_full_paired",64,block bootstrap over same-image caption rows,,25833936,,0,0,2049431,112,"hashed 2-gram, block_size=5000",CI source for prompt-support deltas
cbu_claimed_b64_all7_pair5k,model_based_text_only,claimed controllable basic units,eval_results/raw_summaries/cbu_claimed/claimed_cbu_v2_all7_b64_5k.responses.qwen397_c1024_mt4096.summary.json,surface names encode source dataset,"cc12m_llavanext_paired, cc12m_pixelprose_paired, cc12m_qwen3vl8b_paired, danbooru2023_florence2_paired, datacomp_recap_llava15_paired_url, laion_pop_llama32_paired, pd12m_full_paired",B=64 lexical units,caption record,,69084,14,70000,0,0,14,Qwen3.5-397B-A17B-FP8 via vLLM structured JSON,bad_json_or_failed=916
cbu_claimed_b64_cc12m_three_pair_subset,model_based_text_only,claimed controllable basic units,eval_results/raw_summaries/cbu_claimed/claimed_cbu_v2_cc12m3_b64_5k.responses.qwen397_c1024_mt4096.summary.json,surface names encode source dataset,"cc12m_llavanext_paired, cc12m_pixelprose_paired, cc12m_qwen3vl8b_paired",B=64 lexical units,caption record,,29401,6,30000,0,0,6,Qwen3.5-397B-A17B-FP8 via vLLM structured JSON,bad_json_or_failed=599
cbu_claimed_b64_noncc12m_subset,model_based_text_only,claimed controllable basic units,eval_results/raw_summaries/cbu_claimed/claimed_cbu_v2_completed4_b64_5k.responses.qwen397_c1024_mt4096.summary.json,surface names encode source dataset,"danbooru2023_florence2_paired, datacomp_recap_llava15_paired_url, laion_pop_llama32_paired, pd12m_full_paired",B=64 lexical units,caption record,,39683,8,40000,0,0,8,Qwen3.5-397B-A17B-FP8 via vLLM structured JSON,bad_json_or_failed=317
cbu_claimed_cc12m_bgrid_b16,model_based_text_only,claimed CBU budget sensitivity,eval_results/raw_summaries/cbu_claimed_budget_grid/claimed_cbu_v2_cc12m_four_caption_b16_1k.responses.qwen397_c1024_mt4096.summary.json,CC12M corrected four-caption slice,"ours, Qwen3-VL-8B, LLaVA-NeXT, PixelProse",B=16 lexical units,caption record,1000,3999,4,4000,0,0,4,Qwen3.5-397B-A17B-FP8 via vLLM structured JSON,bad_json_or_failed=1; B-grid raw summary plus plot-ready CSV are copied
cbu_claimed_cc12m_bgrid_b32,model_based_text_only,claimed CBU budget sensitivity,eval_results/raw_summaries/cbu_claimed_budget_grid/claimed_cbu_v2_cc12m_four_caption_b32_1k.responses.qwen397_c1024_mt4096.summary.json,CC12M corrected four-caption slice,"ours, Qwen3-VL-8B, LLaVA-NeXT, PixelProse",B=32 lexical units,caption record,1000,3991,4,4000,0,0,4,Qwen3.5-397B-A17B-FP8 via vLLM structured JSON,bad_json_or_failed=9; B-grid raw summary plus plot-ready CSV are copied
cbu_claimed_cc12m_bgrid_b48,model_based_text_only,claimed CBU budget sensitivity,eval_results/raw_summaries/cbu_claimed_budget_grid/claimed_cbu_v2_cc12m_four_caption_b48_1k.responses.qwen397_c1024_mt4096.summary.json,CC12M corrected four-caption slice,"ours, Qwen3-VL-8B, LLaVA-NeXT, PixelProse",B=48 lexical units,caption record,1000,3980,4,4000,0,0,4,Qwen3.5-397B-A17B-FP8 via vLLM structured JSON,bad_json_or_failed=20; B-grid raw summary plus plot-ready CSV are copied
cbu_claimed_cc12m_bgrid_b64,model_based_text_only,claimed CBU budget sensitivity,eval_results/raw_summaries/cbu_claimed_budget_grid/claimed_cbu_v2_cc12m_four_caption_b64_1k.responses.qwen397_c1024_mt4096.summary.json,CC12M corrected four-caption slice,"ours, Qwen3-VL-8B, LLaVA-NeXT, PixelProse",B=64 lexical units,caption record,1000,3975,4,4000,0,0,4,Qwen3.5-397B-A17B-FP8 via vLLM structured JSON,bad_json_or_failed=25; B-grid raw summary plus plot-ready CSV are copied
cbu_vqa_cc12m_four_caption_b64_4494.responses.gemma4_31b_it_c512_mt2048.summary,image_conditioned_model_based,CBU-derived VQA support/risk,eval_results/raw_summaries/vqa_image_conditioned/cbu_vqa_cc12m_four_caption_b64_4494.responses.gemma4_31b_it_c512_mt2048.summary.json,surface names encode source dataset,"ours_cc12m, ref_cc12m_llavanext, ref_cc12m_qwen3vl8b, ref_pixelprose_cc12m",B=64 lexical units,caption-image record with claimed CBU question set,,17859,4,17859,201065,0,4,Gemma-4-31B-IT via vLLM structured JSON,"bad_json_or_failed=0; support=yes, risk=no, uncertain=uncertain over atomic CBU questions"
cbu_vqa_cc12m_four_caption_llava_url_bridge_b64_4494.responses.qwen397_image_local_c512_mt2048_compact.summary,image_conditioned_model_based,CBU-derived VQA support/risk,eval_results/raw_summaries/vqa_image_conditioned/cbu_vqa_cc12m_four_caption_llava_url_bridge_b64_4494.responses.qwen397_image_local_c512_mt2048_compact.summary.json,surface names encode source dataset,"ours_cc12m, ref_cc12m_llavanext, ref_cc12m_qwen3vl8b, ref_pixelprose_cc12m",B=64 lexical units,caption-image record with claimed CBU question set,,17859,4,17859,201065,0,4,Qwen3.5-397B-A17B-FP8 via vLLM structured JSON,"bad_json_or_failed=0; support=yes, risk=no, uncertain=uncertain over atomic CBU questions"
cbu_vqa_datacomp_b64_5k.summary.qwen397_full_plus_sanity,image_conditioned_model_based,CBU-derived VQA support/risk,eval_results/raw_summaries/vqa_image_conditioned/cbu_vqa_datacomp_b64_5k.summary.qwen397_full_plus_sanity.json,surface names encode source dataset,datacomp_recap_llava15_paired_url,B=64 lexical units,caption-image record with claimed CBU question set,,9639,2,9643,119918,0,2,Qwen3.5-397B-A17B-FP8 via vLLM structured JSON,"bad_json_or_failed=4; support=yes, risk=no, uncertain=uncertain over atomic CBU questions"
cbu_vqa_noncc12m_nondc_b64_5k.summary.qwen397_full,image_conditioned_model_based,CBU-derived VQA support/risk,eval_results/raw_summaries/vqa_image_conditioned/cbu_vqa_noncc12m_nondc_b64_5k.summary.qwen397_full.json,surface names encode source dataset,"danbooru2023_florence2_paired, laion_pop_llama32_paired, pd12m_full_paired",B=64 lexical units,caption-image record with claimed CBU question set,,29812,6,29819,367855,0,6,Qwen3.5-397B-A17B-FP8 via vLLM structured JSON,"bad_json_or_failed=7; support=yes, risk=no, uncertain=uncertain over atomic CBU questions"
caption_embedding_profile_50k,embedding_model_based,"Vendi, covariance effective rank, top eigenvalue mass",eval_results/raw_summaries/embedding_vendi_support/caption_embedding_profile.tsv,paired slices encoded in pair-level summaries,caption surfaces listed per row,full caption as passed to encoder protocol,caption embedding,,400000,8,400000,0,0,8,"BGE-M3-official, E5-Mistral-7B, Qwen3-Embedding-4B, Qwen3-Embedding-8B",50k rows per surface/model where available; encoder-sensitive diversity profile
embedding_prompt_caption_support_50k,embedding_model_based,embedding kNN prompt-to-caption support,eval_results/raw_summaries/embedding_vendi_support/prompt_caption_support.tsv,DataComp prompt/caption support slice,ours vs reference captions,full embedded text under model protocol,prompt embedding to caption embedding gallery,,600000,12,1200000,0,600000,12,"BGE-M3, E5-Mistral, Qwen3-Embedding-4B",kNN support metrics; not used as a scalar caption-quality score
cc12m_longclip_retrieval_full_or_input64,image_text_retrieval_model_based,LongCLIP retrieval margin and Recall@K,eval_results/raw_summaries/longclip_retrieval/longclip_retrieval_summary.json,CC12M corrected four-caption slice,"ours, qwen3vl8b, llavanext, pixelprose",encoder max_length=248; input mode encoded in source paths,image-caption pair,4494,17976,4,22470,0,0,4,zer0int/LongCLIP-GmP-ViT-L-14,retrieval diagnostic only; CLIP-family scores are not treated as a faithfulness certificate