| { |
| "audited_at": "2026-07-26T05:38:59Z", |
| "post_run1_correction": { |
| "active_run1_process_or_shards_changed": false, |
| "behavior": "Known Hub schemas select their real path column and fail closed if it is missing or empty. A leading StarCoderData reponame metadata line is removed only for syntax parsing, while the original text remains the training payload.", |
| "corpus_script": { |
| "path": "scripts/corpus.py", |
| "sha256": "05afba561a6258812d2baa802456c29c00d30955bec7f5c46389b90b2dad9e93" |
| }, |
| "effective_scope": "future_source_streaming_only", |
| "quality_gate_test": { |
| "path": "scripts/test_corpus_quality.py", |
| "sha256": "b90f72e30d0eb5040ded59aaf4ae5a7a4a90c8951bbc66fe4fceb0cdd2ba5eed" |
| } |
| }, |
| "publication_documents": { |
| "data_document": { |
| "path": "DATA.md", |
| "sha256": "fac5dce1b46f29821e3ed509332922ba395d4f3a1660b2e95929e94393e25ec7" |
| }, |
| "model_card_template": { |
| "path": "MODEL_CARD.md", |
| "sha256": "ecd9611c82d8a05da30f36ef41b5ec0642ec7a4e6137c20f5951d84a51d55d9f" |
| } |
| }, |
| "registered_run1_config": { |
| "path": "config/run1.json", |
| "sha256": "f1683a7b00bf1b1e93654ebb366fa0072ba6092b0dbad10272e9353196c3532d" |
| }, |
| "registered_run2_config": { |
| "path": "config/run2_no_fim.json", |
| "sha256": "5692f15944dcef9a8a64a96ea5b5888b39024de321c2bb9a302f1c522bc3ea12" |
| }, |
| "required_publication_disclosures": [ |
| "Do not claim that every Python or JSON training document passed an extension parser.", |
| "Do not claim that the run 1 shards were locally verified as permissive-only.", |
| "State that original StarCoderData repository terms and relevant attribution clauses still apply.", |
| "State that FineWeb-Edu is ODC-By 1.0 and remains subject to Common Crawl terms.", |
| "State that Apache 2.0 covers the Wisp artifact and does not override source-data or generated-code terms.", |
| "State that the tokenizer sampled source entries round-robin by document rather than using the configured training-token weights.", |
| "Do not claim that deterministic run 1 token normalization proves exact original examples or boundaries.", |
| "State that run 1 applied fim_rate 0.7 independently per tokenized chunk, not per source document or sampled training window." |
| ], |
| "run1_fim_application": { |
| "build_log": { |
| "path": "evidence/run1_corpus_build.log", |
| "sha256": "ef5de5c46aac1ff601158b43a3cde481ae6090ea04eefc772c531ff2ac78295e" |
| }, |
| "build_source": { |
| "git_blob_sha1": "18b7e158ecec3467be28e1b18a5bab72c0ee1c77", |
| "git_commit": "a534de4d542167bdcea8adfda8fbf25d6cd0db44", |
| "path": "scripts/prepare_data.py", |
| "sha256": "6ebbd49a92de87582c429e2c0a5e2fd22792b7db1cbf44e37651b4eceaef7ff6" |
| }, |
| "configured_rate_is_per_source_document": false, |
| "configured_rate_is_per_training_window": false, |
| "configured_transform_probability_per_chunk": 0.7, |
| "maximum_chunk_tokens": 1024, |
| "selected_orderings": { |
| "psm_probability": 0.5, |
| "spm_probability": 0.5 |
| }, |
| "selection_unit": "tokenized_chunk", |
| "statement": "Run 1 split each tokenized source document into chunks of at most 1024 tokens and selected FIM independently for each chunk. The configured 0.7 is not a per-document or per-window rate.", |
| "training_window_tokens": 2051 |
| }, |
| "run1_final_build_evidence": { |
| "final_index": { |
| "path": "data/shards/index.json", |
| "sha256": "862b1a9b7cc6c3c0d31299e21b352e2b736de767a99bf7fa38213d6c60fc0db0" |
| }, |
| "log": { |
| "path": "evidence/run1_corpus_build.log", |
| "sha256": "ef5de5c46aac1ff601158b43a3cde481ae6090ea04eefc772c531ff2ac78295e" |
| }, |
| "realized_train_percent": { |
| "fineweb_edu": 7.993028, |
| "implementation_code": 84.997663, |
| "markdown": 7.009309, |
| "starcoderdata_including_markdown": 92.006972 |
| }, |
| "realized_train_tokens": { |
| "HuggingFaceFW/fineweb-edu:sample-10BT": 400290897, |
| "bigcode/starcoderdata:c": 250363883, |
| "bigcode/starcoderdata:go": 451081764, |
| "bigcode/starcoderdata:java": 400525337, |
| "bigcode/starcoderdata:javascript": 650722190, |
| "bigcode/starcoderdata:markdown": 351026251, |
| "bigcode/starcoderdata:python": 1200566506, |
| "bigcode/starcoderdata:rust": 451511705, |
| "bigcode/starcoderdata:shell": 150445965, |
| "bigcode/starcoderdata:sql": 100765073, |
| "bigcode/starcoderdata:typescript": 600701168 |
| }, |
| "scope": "Exact aggregate train-token totals only. Row identities, rejection counts, per-source validation overshoot, and row-level obligations remain unavailable.", |
| "total_train_tokens": 5008000739 |
| }, |
| "run1_provenance_limitations": { |
| "exact_original_unit_boundaries_proven": false, |
| "locally_verified_permissive_only": false, |
| "per_row_attribution_index_available": false, |
| "per_row_license_mapping_preserved": false, |
| "per_source_validation_counts_recorded": false, |
| "realized_per_source_train_tokens_recovered_from_final_log": true, |
| "source_revisions_captured_during_build": false, |
| "source_row_metadata_preserved_in_shards": false, |
| "statement": "Run 1 preserves configured source weights, exact aggregate train-token totals from the recovered final build log, post-build revision evidence, and a post-build hash of every current shard. It does not preserve ordered raw rows, repository paths, rejection counts, per-source validation overshoot, per-row licenses, attribution mapping, or exact original unit boundaries needed for a local permissive-only and example-exact audit." |
| }, |
| "run1_quality_gate": { |
| "configured_path_field": null, |
| "corpus_script_at_build_sha256": "7180e0d69a543fa2ddcf76ef6fa035a14bab2f7e0dfc8a31413b416ad891886e", |
| "extension_parser_activated_for_hub_rows": false, |
| "path_field_used": "path", |
| "reason": "The run 1 iterator requested the absent path column, so Hub rows reached the structural filters without a file extension. The Python ast.parse and JSON json.loads branches therefore did not activate.", |
| "starcoderdata_path_field": "max_stars_repo_path", |
| "structural_filters_applied": true |
| }, |
| "run1_shard_integrity": { |
| "attestation_kind": "post_build_current_bytes_and_visible_grammar", |
| "boundary_recovery": { |
| "detectable_reassembly_groups": 27, |
| "exact_original_units_proven": false, |
| "mode": "deterministic_visible_grammar_normalization", |
| "restored_internal_eos_tokens": 33 |
| }, |
| "normalized_splits": { |
| "train": { |
| "derived_tokens": 4992043184, |
| "fim_units": 5319185, |
| "source_tokens": 5008000739, |
| "units": 7629643 |
| }, |
| "val": { |
| "derived_tokens": 19949502, |
| "fim_units": 18870, |
| "source_tokens": 20006112, |
| "units": 27087 |
| } |
| }, |
| "receipt": { |
| "path": "config/run1_shard_integrity_receipt.json", |
| "sha256": "5831ecd4a471fbe07e19b212bc3de44bed0b0b6b456083e888f66802937bf471" |
| }, |
| "scheduled_run2_training_positions": 4999872512, |
| "source_bytes": 10056013702, |
| "source_files": 52, |
| "source_index_sha256": "862b1a9b7cc6c3c0d31299e21b352e2b736de767a99bf7fa38213d6c60fc0db0" |
| }, |
| "schema_version": 1, |
| "sources": { |
| "HuggingFaceFW/fineweb-edu": { |
| "configured_weight": 0.08, |
| "current_main_revision_at_audit": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", |
| "dataset_card": { |
| "license_label": "odc-by", |
| "sha256": "a0cc8998a20499432b28b6575f3046b714938eb8e11b8d59a1d25ddf3716061e", |
| "terms": "The dataset is distributed under ODC-By 1.0 and remains subject to Common Crawl terms.", |
| "url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu/blob/87f09149ef4734204d70ed1d046ddc9ca3f2b8f9/README.md" |
| }, |
| "observed_stream_row_schema": { |
| "canonical_fields_sha256": "a7b0323d3e758514f936736e75a919bda456e98164299c1c1ce5970f65678f91", |
| "fields": [ |
| "dump", |
| "file_path", |
| "id", |
| "int_score", |
| "language", |
| "language_score", |
| "score", |
| "text", |
| "token_count", |
| "url" |
| ], |
| "subset": "sample-10BT" |
| }, |
| "post_build_cache_ref_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9" |
| }, |
| "bigcode/starcoderdata": { |
| "configured_weight": 0.92, |
| "current_main_revision_at_audit": "9fc30b578cedaec69e47302df72cf00feed7c8c4", |
| "dataset_card": { |
| "license_label": "other", |
| "sha256": "7a3e42cc82fb48b6b81f2ef06eab94af33e605eff743c6a4b8a3b1852ced7c0a", |
| "terms": "Original repository licenses apply, including attribution clauses when relevant. Users must follow the source dataset update and removal terms.", |
| "url": "https://huggingface.co/datasets/bigcode/starcoderdata/blob/9fc30b578cedaec69e47302df72cf00feed7c8c4/README.md" |
| }, |
| "observed_stream_row_schema": { |
| "canonical_fields_sha256": "ddfa03121c2f5e5766eada883df62dcaef2a04540a49d849692831ae81fbddd4", |
| "fields": [ |
| "content", |
| "id", |
| "max_stars_count", |
| "max_stars_repo_name", |
| "max_stars_repo_path" |
| ], |
| "subset": "python" |
| }, |
| "post_build_cache_ref_revision": "9fc30b578cedaec69e47302df72cf00feed7c8c4" |
| } |
| }, |
| "status": "limitations_registered", |
| "tokenizer_sampling": { |
| "build_log": { |
| "path": "evidence/tokenizer_build.log", |
| "sha256": "7c28f91dc527e0cc37d23c520ba183aef845f948b64ea822f3b3ca17de264467" |
| }, |
| "configured_token_weights_applied": false, |
| "documents_requested": 400000, |
| "exact_row_manifest_preserved": false, |
| "source_entries": 11, |
| "statement": "The tokenizer sample included FineWeb-Edu and sampled source entries evenly by document, not according to the later training-token weights.", |
| "strategy": "round_robin_by_configured_source_entry", |
| "tokenizer": { |
| "path": "tokenizer/code32k.json", |
| "sha256": "401a28c1f079050c48f6438830ca772d161d897e3cf2f30588d9ddc587dc6081" |
| } |
| } |
| } |
|
|
