social-data-attribution-w2/docs/data_home/inventory.json

{
  "schema_version": "1",
  "generated_at": "2026-05-22",
  "generated_by": "preservation-gap-audit and PACE-to-cloud migration",
  "notes": "Single-source-of-truth inventory of Dolma3 data attribution artifacts across HuggingFace (HCAI-Lab org) and Cloudflare R2 (soc127-dedup bucket). One entry per artifact group. See README.md for navigation by use case.",
  "r2": {
    "endpoint": "https://0934ab8e84ac8f4e81decaf3eb121337.r2.cloudflarestorage.com",
    "bucket": "soc127-dedup",
    "credential_source": "1Password \u2192 Research / Cloudflare R2 API Credentials (scoped tokens issued per consumer team)"
  },
  "artifacts": [
    {
      "id": "dolma3_dedup_corpus",
      "purpose": "Deduplicated 6T-token Dolma3 source corpus (the raw documents)",
      "kind": "hf-bucket-and-r2-prefix",
      "location": [
        {
          "repo_id": "HCAI-Lab/dolma3-6t-corpus-raw",
          "kind": "bucket",
          "layout": "phase1_pool_shared/ and phase2_nonpool_final/ folders (R2 keys with the soc127/ prefix stripped)",
          "migration_status": "in_progress_2026-05-29 (8-shard SLURM array 5342434; ~175916 objects / 3.4 TB; byte-verification pending before R2 deletion)"
        },
        {
          "bucket": "soc127-dedup",
          "kind": "r2-prefix",
          "prefixes": [
            "soc127/phase1_pool_shared/",
            "soc127/phase2_nonpool_final/"
          ],
          "note": "Original R2 source. To be deleted only after the HF bucket mirror is byte-verified (pending user approval)."
        }
      ],
      "scale": {
        "shards": 58621,
        "objects": 175916,
        "approx_tokens": "6T",
        "approx_size_gb": 3400,
        "shard_format": ".jsonl.zst (+ .done / .stats.json markers)"
      },
      "schema_ref": "docs/DATA_INVENTORY.md \u00a7Source corpus, \u00a7Document format",
      "soc_lineage": [
        "SOC-127"
      ],
      "public_visibility": "private (HF bucket + R2, credentialed read)",
      "consumer_use_case": "Run attribution / training / analysis on raw source documents.",
      "collection": "Source Corpus + Manifest"
    },
    {
      "id": "dedup_intermediates",
      "purpose": "Intermediate dedup-pipeline outputs (phase2 bucketed + raw per-source files, manifests, pipeline reports). Reproducible from the source pool + bloom filter; archived for auditability.",
      "kind": "hf-bucket",
      "location": [
        {
          "repo_id": "HCAI-Lab/archive-dolma3-6t-dedup-intermediates",
          "kind": "bucket",
          "layout": "per-bucket tar.zst bundles (one per phase2_nonpool_bucketed/bucket_NNNN, plus phase2_nonpool_raw / manifests / _root); ~261 tarballs",
          "migration_status": "in_progress_2026-05-29 (8-shard SLURM array 5342752; ~4.18M source objects / 390 GB tarballed; verification pending)"
        },
        {
          "bucket": "soc127-dedup",
          "kind": "r2-prefix",
          "prefixes": [
            "soc127/phase2_nonpool_bucketed/",
            "soc127/phase2_nonpool_raw/",
            "soc127/manifests/"
          ],
          "note": "Original R2 source (the bare soc127/ prefix minus the final corpus dirs). Delete only after tarball archive is verified (pending user approval)."
        }
      ],
      "scale": {
        "source_objects": 4184032,
        "approx_size_gb": 390,
        "tarballs": 261
      },
      "soc_lineage": [
        "SOC-127"
      ],
      "public_visibility": "private HF bucket",
      "consumer_use_case": "Audit or re-derive dedup decisions; not needed for normal workflows (final corpus is dolma3-6t-corpus-raw).",
      "collection": "Archive (pre-6T and legacy)"
    },
    {
      "id": "weborganizer_labels",
      "purpose": "Per-document WebOrganizer topic + format labels (24 topics \u00d7 24 formats, 576 bins)",
      "kind": "hf-dataset-and-r2-prefix",
      "location": [
        {
          "repo_id": "HCAI-Lab/soc91-labels",
          "kind": "dataset",
          "scale": {
            "files": 2719,
            "approx_size_gb": 169.94,
            "format": ".parquet (~60 MB chunks)"
          },
          "note": "HF mirror: 58465 source per-shard parquets concatenated into 2719 ~60MB chunks. Each row carries source_shard_path. README hand-written after script's post-upload write_readme() crash (data itself uploaded cleanly)."
        },
        {
          "bucket": "soc127-dedup",
          "prefixes": [
            "soc91-labels/"
          ],
          "scale": {
            "files": 58465,
            "format": ".parquet (one per source shard) + .stats.json"
          }
        }
      ],
      "schema_ref": "src/dolma/sidecar_manifest_fields.py:43-124 + README \u00a7Sidecar corpus manifest",
      "soc_lineage": [
        "SOC-91",
        "SOC-142 (quality-label fix)"
      ],
      "public_visibility": "private HF dataset + private R2 (both credentialed read)",
      "consumer_use_case": "Filter or stratify documents by topic/format without re-running the classifier. Prefer HF for typical use; R2 prefix preserves per-source-shard granularity if needed.",
      "uploaded_at": "2026-05-23 (HF mirror)",
      "collection": "Source Corpus + Manifest"
    },
    {
      "id": "weborganizer_eda",
      "purpose": "SOC-101 sidecar EDA validation outputs (audit reports, label-distribution stats)",
      "kind": "hf-dataset-and-r2-prefix",
      "location": [
        {
          "repo_id": "HCAI-Lab/soc91-stats",
          "kind": "dataset",
          "note": "Raw R2 mirror of the EDA artifact tree (JSON reports + stats); not Parquet-converted because contents are mostly small JSON files."
        },
        {
          "bucket": "soc127-dedup",
          "prefixes": [
            "soc91-stats/"
          ]
        }
      ],
      "schema_ref": "docs/SOC95_MODAL_RUNBOOK.md \u00a7validation; scripts/modal/sidecar_eda.py",
      "soc_lineage": [
        "SOC-101"
      ],
      "public_visibility": "private HF dataset + private R2",
      "consumer_use_case": "Reference for label-distribution validation; not normally needed by external users.",
      "uploaded_at": "2026-05-23 (HF mirror)",
      "collection": "Source Corpus + Manifest"
    },
    {
      "id": "quality_sidecars",
      "purpose": "Per-doc quality score sidecars (post-SOC-142 label-fix)",
      "kind": "hf-dataset-and-r2-prefix",
      "location": [
        {
          "repo_id": "HCAI-Lab/soc139-quality-sidecars",
          "kind": "dataset",
          "scale": {
            "rows": 1257700000,
            "files": 80,
            "approx_size_gb": 41.8,
            "columns": [
              "doc_id",
              "quality_label_id",
              "quality_score",
              "quality_high_prob",
              "quality_low_prob",
              "quality_confidence",
              "source_shard_path"
            ]
          },
          "note": "HF mirror: per-shard parquets concatenated into 80 chunks (~500MB each)."
        },
        {
          "bucket": "soc127-dedup",
          "prefixes": [
            "soc139-quality-sidecars/"
          ],
          "scale": {
            "files": 58621,
            "format": ".parquet + .label-fix.done + .stats.json"
          }
        }
      ],
      "schema_ref": "src/dolma/quality/sidecar.py (QualityShardStats); src/dolma/sidecar_manifest_fields.py merge logic",
      "soc_lineage": [
        "SOC-139",
        "SOC-142 (inverted-label fix at commit 3342baf)"
      ],
      "public_visibility": "private HF dataset + private R2",
      "consumer_use_case": "Quality-filter the corpus by score threshold. Prefer HF for typical use.",
      "uploaded_at": "2026-05-23 (HF mirror)",
      "collection": "Source Corpus + Manifest"
    },
    {
      "id": "corpus_manifest_dataset",
      "purpose": "Unified per-document manifest joining topic/format + quality + tokens + source shard path. 1.1B docs, 32-column PyArrow schema.",
      "kind": "hf-dataset",
      "location": [
        {
          "repo_id": "HCAI-Lab/dolma3-olmo3-corpus-manifest",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/dolma3-6t-corpus-manifest",
          "kind": "dataset"
        },
        {
          "bucket": "soc127-dedup",
          "prefix": "soc95-manifest/"
        }
      ],
      "scale": {
        "approx_docs": 1098646162,
        "approx_tokens": 2116727590753,
        "approx_size_gb": 213
      },
      "schema_ref": "src/data_attribution/recipes/corpus_manifest.py:15-52",
      "soc_lineage": [
        "SOC-95"
      ],
      "public_visibility": "public HF dataset",
      "consumer_use_case": "One-stop manifest for filtering/stratifying without joining sidecars yourself.",
      "collection": "Source Corpus + Manifest",
      "old_names": [
        "HCAI-Lab/dolma3_6T_corpus_manifest",
        "HCAI-Lab/dolma3_olmo3_corpus_manifest"
      ],
      "renamed_at": "2026-05-25"
    },
    {
      "id": "stratified_samples",
      "purpose": "Materialized stratified working samples (N docs/bin across 576 bins, seed=42)",
      "kind": "hf-multi-dataset",
      "location": [
        {
          "repo_id": "HCAI-Lab/dolma3-6t-sample-500-docs",
          "kind": "dataset",
          "docs": 287936,
          "tokens": 538830013
        },
        {
          "repo_id": "HCAI-Lab/dolma3-6t-sample-1000-docs",
          "kind": "dataset",
          "docs": 575187,
          "tokens": 1083585221
        },
        {
          "repo_id": "HCAI-Lab/dolma3-6t-sample-5000-docs",
          "kind": "dataset",
          "docs": 2855446,
          "tokens": 5299479330
        },
        {
          "repo_id": "HCAI-Lab/dolma3-6t-sample-10000-docs",
          "kind": "dataset",
          "docs": 5678621,
          "tokens": 10506536366
        },
        {
          "repo_id": "HCAI-Lab/dolma3-6t-sample-50000-docs",
          "kind": "dataset",
          "docs": 26249124,
          "tokens": 62819501017
        },
        {
          "repo_id": "HCAI-Lab/dolma3-6t-sample-100000-docs",
          "kind": "dataset",
          "docs": 49709294,
          "tokens": 118399343310,
          "underfilled_bins": 130,
          "twin_bucket": "HCAI-Lab/dolma3-6t-sample-100000-docs",
          "created_at": "2026-05-25"
        },
        {
          "repo_id": "HCAI-Lab/dolma3-6t-sample-200000-docs",
          "kind": "bucket",
          "files": 58062,
          "bytes_approx_gb": 310.6,
          "created_at": "2026-05-29",
          "note": "200k docs/bin sample. Migrated from R2 soc134-samples/sample_200000_docs/ on 2026-05-29 (HF bucket only). Count-verified: HF 58062 = R2 58062. Doc/token totals not recomputed (the R2 staging prefix had no bin_summary.csv)."
        }
      ],
      "schema_ref": "docs/WORKING_SAMPLE_DATA_ACCESS.md \u00a7Manifest schema",
      "soc_lineage": [
        "SOC-134"
      ],
      "public_visibility": "public HF datasets",
      "consumer_use_case": "Stratified subsets for training, evaluation, or analysis without materializing your own sample.",
      "collection": "Working Samples + Preconditioner",
      "old_names": [
        "HCAI-Lab/dolma3_6T_sample_10000_docs",
        "HCAI-Lab/dolma3_6T_sample_1000_docs",
        "HCAI-Lab/dolma3_6T_sample_50000_docs",
        "HCAI-Lab/dolma3_6T_sample_5000_docs",
        "HCAI-Lab/dolma3_6T_sample_500_docs"
      ],
      "renamed_at": "2026-05-25"
    },
    {
      "id": "preconditioner_sample",
      "purpose": "100K uniform-random preconditioner sample (no stratification)",
      "kind": "hf-dataset",
      "location": [
        {
          "repo_id": "HCAI-Lab/dolma3-6t-preconditioner-100k",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/dolma3-6t-preconditioner-100k",
          "kind": "bucket"
        }
      ],
      "scale": {
        "docs": 100000,
        "tokens": 251517816,
        "unique_source_shards": 38277,
        "min_tokens_per_doc": 512
      },
      "schema_ref": "docs/WORKING_SAMPLE_DATA_ACCESS.md \u00a7Preconditioner sample",
      "soc_lineage": [
        "SOC-151"
      ],
      "public_visibility": "public HF dataset + public HF bucket",
      "consumer_use_case": "Sample for TrackStar / EK-FAC preconditioner construction.",
      "collection": "Working Samples + Preconditioner",
      "old_names": [
        "HCAI-Lab/dolma3_6T_preconditioner_100k",
        "HCAI-Lab/preconditioner-100k"
      ],
      "renamed_at": "2026-05-25"
    },
    {
      "id": "trackstar_gradient_index_base",
      "purpose": "Per-document gradient index built with Bergson from sample_10000_docs (5.68M docs). 316 shard subdirs, each with gradients.bin, normalizers, preconditioners, configs.",
      "kind": "hf-bucket",
      "location": [
        {
          "repo_id": "HCAI-Lab/trackstar-gradient-index-base",
          "kind": "bucket"
        }
      ],
      "old_names": [
        "HCAI-Lab/tracstar-index-base-soc156"
      ],
      "scale": {
        "shards": 316,
        "approx_size_tb": 1.2,
        "model": "allenai/Olmo-3-1025-7B (base)",
        "projection_dim": 16,
        "precision": "fp32"
      },
      "schema_ref": "docs/TRACKSTAR_DATA_ARTIFACTS.md \u00a75 \u00abGradient Index\u00bb; bergson library docs at github.com/EleutherAI/bergson",
      "soc_lineage": [
        "SOC-156"
      ],
      "public_visibility": "private HF bucket",
      "consumer_use_case": "Score new query sets against the existing training corpus without rebuilding gradients. ~156 GPU-hours to reproduce.",
      "uploaded_at": "2026-05-22",
      "renamed_at": "2026-05-24",
      "collection": "TrackStar Indices + Training Shards"
    },
    {
      "id": "trackstar_training_shards",
      "purpose": "TrackStar training shards with positional doc IDs (id+text only). 316 JSONL files mapping shard_NNNN:INDEX to original Dolma UUID + text.",
      "kind": "hf-dataset",
      "location": [
        {
          "repo_id": "HCAI-Lab/dolma3-6t-sample-10000-docs-trackstar-shards",
          "kind": "dataset"
        }
      ],
      "old_names": [
        "HCAI-Lab/dolma3_6T_sample_10000_docs",
        "HCAI-Lab/dolma3_6T_sample_10000_docs_trackstar_shards",
        "HCAI-Lab/dolma3_6T_sample_10000_docs_tracstar_shards"
      ],
      "scale": {
        "files": 316,
        "approx_size_gb": 41,
        "format": ".jsonl"
      },
      "schema_ref": "docs/TRACKSTAR_DATA_ARTIFACTS.md \u00a73 \u00abDoc ID Mapping\u00bb",
      "soc_lineage": [
        "SOC-156"
      ],
      "public_visibility": "private HF dataset",
      "consumer_use_case": "REQUIRED to resolve positional doc IDs in cloud score matrices back to source text.",
      "uploaded_at": "2026-05-22",
      "renamed_at": "2026-05-25",
      "collection": "TrackStar Indices + Training Shards"
    },
    {
      "id": "trackstar_query_gradients_base",
      "purpose": "Per-query gradient builds for the four OLMES benchmarks (and BBH / ARC for SOC-170 / SOC-171). One Bergson-style index per benchmark dir.",
      "kind": "hf-bucket",
      "location": [
        {
          "repo_id": "HCAI-Lab/trackstar-query-gradients-base",
          "kind": "bucket"
        }
      ],
      "old_names": [
        "HCAI-Lab/tracstar-query-builds-soc156"
      ],
      "scale": {
        "variants": [
          "base",
          "instruct_base",
          "instruct_cot"
        ],
        "approx_size_gb": 17
      },
      "schema_ref": "docs/TRACKSTAR_DATA_ARTIFACTS.md \u00a76 \u00abQuery Gradient Indices\u00bb",
      "soc_lineage": [
        "SOC-156",
        "SOC-170",
        "SOC-171"
      ],
      "public_visibility": "private HF bucket",
      "consumer_use_case": "Optional. Needed if scoring additional training data against existing query sets.",
      "uploaded_at": "2026-05-22",
      "renamed_at": "2026-05-24",
      "collection": "TrackStar Indices + Training Shards"
    },
    {
      "id": "trackstar_preconditioners",
      "purpose": "TrackStar mixed preconditioners for OLMo3 base, instruct, and think model variants.",
      "kind": "hf-bucket",
      "location": [
        {
          "repo_id": "HCAI-Lab/trackstar-preconditioners",
          "kind": "bucket"
        }
      ],
      "old_names": [
        "HCAI-Lab/tracstar-preconditioners"
      ],
      "scale": {
        "models": 3,
        "approx_size_mb": 885,
        "files": 78
      },
      "soc_lineage": [
        "SOC-152",
        "SOC-168"
      ],
      "public_visibility": "public HF bucket",
      "consumer_use_case": "Apply pre-built preconditioner to new attribution scoring runs. NOTE: preconditioner must match the model used for gradient build (SOC-162 finding).",
      "renamed_at": "2026-05-24",
      "collection": "TrackStar Indices + Training Shards"
    },
    {
      "id": "trackstar_scores_base_olmes_4bench",
      "purpose": "Per-query attribution score matrices for the OLMo-3-7B base run, 4 OLMES benchmarks (gsm8k, mmlu_social_science, mmlu_stem, socialiqa) \u00d7 316 shards = 2532 .npy files.",
      "kind": "hf-bucket",
      "location": [
        {
          "repo_id": "HCAI-Lab/trackstar-scores-base-olmes-4bench",
          "kind": "bucket"
        }
      ],
      "old_names": [
        "HCAI-Lab/tracstar-scores-base-soc156"
      ],
      "scale": {
        "benchmarks": [
          "gsm8k",
          "mmlu_social_science",
          "mmlu_stem",
          "socialiqa"
        ],
        "files": 2532,
        "approx_size_gb": 396,
        "model": "allenai/Olmo-3-1025-7B (base)"
      },
      "schema_ref": "docs/TRACKSTAR_DATA_ARTIFACTS.md \u00a71 \u00abFull Score Matrices\u00bb",
      "soc_lineage": [
        "SOC-156"
      ],
      "public_visibility": "public HF bucket",
      "consumer_use_case": "Influence(doc_i, query_j) lookup for the four OLMES benchmarks on OLMo-3-7B base.",
      "renamed_at": "2026-05-24",
      "collection": "TrackStar Scores + Analysis"
    },
    {
      "id": "trackstar_scores_instruct_cot_olmes_4bench",
      "purpose": "Score matrices for the OLMo-3-7B instruct-cot (think) run, 4 OLMES benchmarks (same shape as base).",
      "kind": "hf-bucket",
      "location": [
        {
          "repo_id": "HCAI-Lab/trackstar-scores-instruct-cot-olmes-4bench",
          "kind": "bucket"
        }
      ],
      "old_names": [
        "HCAI-Lab/tracstar-scores-instruct-cot-soc161"
      ],
      "scale": {
        "files": 2532,
        "approx_size_gb": 396,
        "model": "allenai/Olmo-3-1025-7B-think"
      },
      "soc_lineage": [
        "SOC-161"
      ],
      "public_visibility": "public HF bucket",
      "consumer_use_case": "Same as base scores but for the instruct-cot model variant.",
      "renamed_at": "2026-05-24",
      "collection": "TrackStar Scores + Analysis"
    },
    {
      "id": "trackstar_scores_instruct_cot_noprecond_mmlu_ss",
      "purpose": "Score matrices for the OLMo-3-7B instruct-cot run with NO preconditioner applied (single benchmark: MMLU social science only). Ablation comparison to the preconditioned scores in the parent OLMES-4bench bucket.",
      "kind": "hf-bucket",
      "location": [
        {
          "repo_id": "HCAI-Lab/trackstar-scores-instruct-cot-noprecond-mmlu-ss",
          "kind": "bucket"
        }
      ],
      "old_names": [
        "HCAI-Lab/tracstar-scores-instruct-cot-noprecond-soc161"
      ],
      "scale": {
        "benchmarks": [
          "mmlu_social_science"
        ],
        "files": 633,
        "approx_size_gb": 70.0,
        "model": "allenai/Olmo-3-1025-7B-think",
        "note": "316 shard_NNNN.npy (221 MB each) + 316 shard_NNNN_doc_ids.json + 1 query_ids.json"
      },
      "schema_ref": "docs/TRACKSTAR_DATA_ARTIFACTS.md \u00a71 \u00abFull Score Matrices\u00bb",
      "soc_lineage": [
        "SOC-161 (noprecond ablation variant)"
      ],
      "public_visibility": "public HF bucket",
      "consumer_use_case": "Compare preconditioned vs raw influence scores for the same model+benchmark to study preconditioner contribution.",
      "uploaded_at": "2026-05-23",
      "renamed_at": "2026-05-24",
      "collection": "TrackStar Scores + Analysis"
    },
    {
      "id": "trackstar_scores_bbh",
      "purpose": "BBH attribution scores for base and instruct_base model variants.",
      "kind": "hf-bucket",
      "location": [
        {
          "repo_id": "HCAI-Lab/trackstar-scores-base-bbh",
          "kind": "bucket"
        },
        {
          "repo_id": "HCAI-Lab/trackstar-scores-instruct-base-bbh",
          "kind": "bucket"
        }
      ],
      "old_names": [
        "HCAI-Lab/tracstar-scores-base-soc170",
        "HCAI-Lab/tracstar-scores-instruct_base-soc170"
      ],
      "scale": {
        "files_each": 1899,
        "approx_size_gb_each": 14
      },
      "soc_lineage": [
        "SOC-170"
      ],
      "public_visibility": "public HF buckets",
      "consumer_use_case": "BBH-task attribution analysis.",
      "renamed_at": "2026-05-24",
      "collection": "TrackStar Scores + Analysis"
    },
    {
      "id": "trackstar_scores_gsm8k_arc",
      "purpose": "GSM8K + ARC attribution scores for base and instruct_base.",
      "kind": "hf-bucket",
      "location": [
        {
          "repo_id": "HCAI-Lab/trackstar-scores-base-gsm8k-arc",
          "kind": "bucket"
        },
        {
          "repo_id": "HCAI-Lab/trackstar-scores-instruct-base-gsm8k-arc",
          "kind": "bucket"
        }
      ],
      "old_names": [
        "HCAI-Lab/tracstar-scores-base-soc171",
        "HCAI-Lab/tracstar-scores-instruct_base-soc171"
      ],
      "scale": {
        "files_each": 1899,
        "approx_size_gb_each": 79
      },
      "soc_lineage": [
        "SOC-171"
      ],
      "public_visibility": "public HF buckets",
      "renamed_at": "2026-05-24",
      "collection": "TrackStar Scores + Analysis"
    },
    {
      "id": "top_k_results",
      "purpose": "Ranked top-K influence results (top-100 JSONL and top-2K parquet/CSV) per benchmark.",
      "kind": "hf-mixed",
      "location": [
        {
          "repo_id": "HCAI-Lab/dolma3-trackstar-influence-scores",
          "kind": "dataset",
          "private": true,
          "files": [
            "influence_scores_full.parquet",
            "top2k_{gsm8k,mmlu_socsci,mmlu_stem,socialiqa}.{csv,parquet}"
          ]
        },
        {
          "repo_id": "HCAI-Lab/trackstar-top2k-base-gsm8k-arc",
          "kind": "bucket",
          "soc": "SOC-171"
        },
        {
          "repo_id": "HCAI-Lab/trackstar-top2k-instruct-base-gsm8k-arc",
          "kind": "bucket",
          "soc": "SOC-171"
        }
      ],
      "old_names": [
        "HCAI-Lab/dolma3-tracstar-influence-scores",
        "HCAI-Lab/tracstar-top2k-soc171-base",
        "HCAI-Lab/tracstar-top2k-soc171-instruct-base"
      ],
      "schema_ref": "docs/TRACKSTAR_DATA_ARTIFACTS.md \u00a72 \u00abTop-100 Ranked Results\u00bb",
      "soc_lineage": [
        "SOC-156",
        "SOC-164",
        "SOC-171"
      ],
      "consumer_use_case": "Drop-in ranked influence results for the four canonical OLMES benchmarks; cheapest path for analysis that only needs top-K docs per query.",
      "renamed_at": "2026-05-24",
      "collection": "TrackStar Scores + Analysis"
    },
    {
      "id": "query_data",
      "purpose": "OLMES evaluation queries used for attribution (prompt + answer + correctness label).",
      "kind": "hf-multi-dataset",
      "location": [
        {
          "repo_id": "HCAI-Lab/base-query-data",
          "kind": "dataset",
          "model": "OLMo-3-7B base"
        },
        {
          "repo_id": "HCAI-Lab/instruct-query-data",
          "kind": "dataset",
          "model": "OLMo-3-7B instruct"
        },
        {
          "repo_id": "HCAI-Lab/instruct-cot-query-data",
          "kind": "dataset",
          "model": "OLMo-3-7B instruct-cot"
        },
        {
          "repo_id": "HCAI-Lab/archive-reduced-queries",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/archive-common-sense-queries",
          "kind": "dataset"
        }
      ],
      "scale": {
        "benchmarks": [
          "gsm8k:1319",
          "mmlu_social_science:3077",
          "mmlu_stem:3018",
          "socialiqa:10000"
        ]
      },
      "schema_ref": "docs/TRACKSTAR_DATA_ARTIFACTS.md \u00a74 \u00abQuery ID Mapping\u00bb",
      "soc_lineage": [
        "SOC-119",
        "SOC-155 (LLM-verified SocialIQA reasoning types)"
      ],
      "public_visibility": "public HF datasets",
      "consumer_use_case": "Reuse the same query sets for new attribution runs to keep results comparable.",
      "collection": "Query Data",
      "old_names": [
        "HCAI-Lab/common-sense-queries",
        "HCAI-Lab/reduced-queries"
      ],
      "renamed_at": "2026-05-25"
    },
    {
      "id": "olmes_eval_results",
      "purpose": "OLMES benchmark evaluation results across model variants.",
      "kind": "hf-multi-dataset",
      "location": [
        {
          "repo_id": "HCAI-Lab/olmes-eval-olmo3-7b-base",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/olmes-eval-olmo3-7b-instruct-base",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/olmes-eval-olmo3-7b-instruct-cot",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/olmes-eval-olmo3-7b-thinking",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/olmes-eval-olmo3-7b-think-mc",
          "kind": "dataset",
          "private": true
        },
        {
          "repo_id": "HCAI-Lab/olmes-eval-olmo3-7b-think-cot",
          "kind": "dataset",
          "private": true
        },
        {
          "repo_id": "HCAI-Lab/olmes-eval-smollm3-3b-base",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/trackstar-olmes-eval-artifacts",
          "kind": "bucket",
          "soc": "SOC-166"
        }
      ],
      "old_names": [
        "HCAI-Lab/data-attribution-olmo-3-7B-base-evaluation",
        "HCAI-Lab/data-attribution-olmo-3-7B-instruct-base-evaluation",
        "HCAI-Lab/data-attribution-olmo-3-7B-instruct-cot-evaluation",
        "HCAI-Lab/data-attribution-olmo-3-7B-think-cot-evaluation",
        "HCAI-Lab/data-attribution-olmo-3-7B-think-mc-evaluation",
        "HCAI-Lab/data-attribution-olmo-3-7B-thinking-evaluation",
        "HCAI-Lab/data-attribution-smollm3-3b-base-evaluation",
        "HCAI-Lab/soc166-olmes-eval-artifacts"
      ],
      "soc_lineage": [
        "SOC-166",
        "SOC-167"
      ],
      "consumer_use_case": "Inspect per-query model predictions and correctness used in attribution analysis.",
      "renamed_at": "2026-05-25",
      "collection": "OLMES Evaluations"
    },
    {
      "id": "dedup_state",
      "purpose": "Deduplication artifacts: Bloom filter, doc IDs, unique docs materialization.",
      "kind": "hf-multi-dataset",
      "location": [
        {
          "repo_id": "HCAI-Lab/dolma3-6t-bloom-index",
          "kind": "dataset",
          "purpose": "Bloom filter (~unique IDs)"
        },
        {
          "repo_id": "HCAI-Lab/archive-dolma3-6t-doc-ids-pershard",
          "kind": "dataset",
          "purpose": "Per-shard doc ID lists (gzip JSONL)",
          "private": true
        },
        {
          "repo_id": "HCAI-Lab/archive-dolma3-6t-mix-doc-ids",
          "kind": "dataset",
          "purpose": "Mix doc IDs",
          "private": true
        },
        {
          "repo_id": "HCAI-Lab/dolma3-6t-unique",
          "kind": "dataset",
          "purpose": "1.258B unique docs materialized"
        },
        {
          "repo_id": "HCAI-Lab/archive-dolma3-6t-dedup-state",
          "kind": "dataset",
          "purpose": "Bloom + dedup backup",
          "private": true
        }
      ],
      "schema_ref": "src/dolma/provenance.py",
      "soc_lineage": [
        "SOC-90",
        "SOC-127"
      ],
      "consumer_use_case": "Reference for dedup decisions; reproduce the dedup pipeline.",
      "collection": "Archive (pre-6T and legacy)",
      "old_names": [
        "HCAI-Lab/dolma3_6t_doc_ids",
        "HCAI-Lab/dolma3_6t_mix_doc_ids",
        "HCAI-Lab/dolma3_6t_unique",
        "HCAI-Lab/soc-90-keep-backup"
      ],
      "renamed_at": "2026-05-25"
    },
    {
      "id": "earlier_enrichment_pool",
      "purpose": "Pre-6T enrichment work: 150B pool sample plus enriched/cleaned variants.",
      "kind": "hf-multi-dataset",
      "location": [
        {
          "repo_id": "HCAI-Lab/archive-dolma3-pool-150b",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/archive-dolma3-pool-150b-cleaned",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/archive-dolma3-pool-150b-enriched",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/archive-dolma3-pool-150b-samples",
          "kind": "dataset",
          "private": true
        },
        {
          "repo_id": "HCAI-Lab/archive-dolma3-pool-stratified",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/archive-dolma3-mix-150b-enriched",
          "kind": "dataset",
          "private": true
        },
        {
          "repo_id": "HCAI-Lab/archive-dolma3-enriched-100k",
          "kind": "dataset"
        },
        {
          "repo_id": "HCAI-Lab/archive-dolma3-enriched-small",
          "kind": "dataset"
        }
      ],
      "soc_lineage": [
        "SOC-5",
        "SOC-7"
      ],
      "consumer_use_case": "Reference for earlier enrichment / sampling work. Not needed for current attribution use.",
      "collection": "Archive (pre-6T and legacy)",
      "old_names": [
        "HCAI-Lab/dolma3-enriched-100k",
        "HCAI-Lab/dolma3_mix_150B_enriched",
        "HCAI-Lab/dolma3_pool_150B",
        "HCAI-Lab/dolma3_pool_150B_cleaned",
        "HCAI-Lab/dolma3_pool_150B_enriched",
        "HCAI-Lab/dolma3_pool_150B_samples",
        "HCAI-Lab/dolma3_pool_stratified",
        "HCAI-Lab/dolma_enriched_small"
      ],
      "renamed_at": "2026-05-25"
    },
    {
      "id": "visualization_artifacts",
      "purpose": "Figures and analysis outputs (heatmaps, bin-characterization, paper figures).",
      "kind": "hf-multi-bucket",
      "location": [
        {
          "repo_id": "HCAI-Lab/dolma3-bin-characterization",
          "kind": "bucket"
        },
        {
          "repo_id": "HCAI-Lab/trackstar-bbh-eval-artifacts",
          "kind": "bucket"
        },
        {
          "repo_id": "HCAI-Lab/trackstar-figures-gsm8k-arc",
          "kind": "bucket"
        },
        {
          "repo_id": "HCAI-Lab/trackstar-analysis-gsm8k-arc",
          "kind": "bucket"
        },
        {
          "repo_id": "HCAI-Lab/dolma3-influence-heatmaps",
          "kind": "dataset"
        }
      ],
      "old_names": [
        "HCAI-Lab/soc14-rq4-bin-characterization",
        "HCAI-Lab/soc170-bbh-artifacts",
        "HCAI-Lab/tracstar-figures-soc171",
        "HCAI-Lab/tracstar-analysis-soc171",
        "HCAI-Lab/soc28-influence-heatmaps"
      ],
      "soc_lineage": [
        "SOC-14",
        "SOC-28",
        "SOC-170",
        "SOC-171"
      ],
      "renamed_at": "2026-05-24",
      "collection": "TrackStar Scores + Analysis"
    },
    {
      "id": "job_archive",
      "purpose": "Archival snapshot of PACE SLURM jobs, logs, dedup state, and small ancillary work outputs.",
      "kind": "hf-dataset",
      "location": [
        {
          "repo_id": "HCAI-Lab/dolma3-attribution-job-archive",
          "kind": "dataset"
        }
      ],
      "soc_lineage": [
        "SOC-14",
        "SOC-90",
        "SOC-149",
        "SOC-159",
        "SOC-161",
        "SOC-167",
        "SOC-171"
      ],
      "public_visibility": "private HF dataset",
      "consumer_use_case": "Reproducibility audits. Not normally needed.",
      "uploaded_at": "2026-05-22",
      "collection": "Archive (pre-6T and legacy)"
    }
  ],
  "schema_references": {
    "source_shard_jsonl": "docs/DATA_INVENTORY.md \u00a7Document format",
    "working_sample_manifest": "docs/WORKING_SAMPLE_DATA_ACCESS.md \u00a7Manifest schema (6 columns: doc_id, token_count, shard_path, bin_id, bin_topic, bin_format)",
    "corpus_manifest_32col": "src/data_attribution/recipes/corpus_manifest.py lines 15-52",
    "sidecar_row_construction": "src/dolma/sidecar_manifest_fields.py lines 43-124",
    "score_matrix_format": "docs/TRACKSTAR_DATA_ARTIFACTS.md \u00a71",
    "bin_id_formula": "topic_idx * len(FORMATS) + format_idx + 1 (576 bins, 1-indexed); src/dolma/manifest_fields.py:60-67"
  },
  "code_repo": "https://github.com/eilab-gt/social-data-attribution",
  "credentials": {
    "hf": "Use HF_TOKEN or a local token file on managed compute. Repository administrators grant collaborator access for private repos.",
    "r2": "Read-only API token via 1Password Share. R2_ACCESS_KEY_ID + R2_SECRET_ACCESS_KEY env vars. scripts/bootstrap/with_r2_credentials.sh wraps the lookup.",
    "pace": "Not granted to external consumers. PACE access requires Georgia Tech sponsorship."
  }
}