| """Fail-closed checks for Wisp's training-data disclosure receipt.""" |
|
|
| import hashlib |
| import json |
| import os |
|
|
|
|
| EXPECTED_SOURCE_REVISIONS = { |
| "bigcode/starcoderdata": "9fc30b578cedaec69e47302df72cf00feed7c8c4", |
| "HuggingFaceFW/fineweb-edu": ( |
| "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9" |
| ), |
| } |
| EXPECTED_SOURCES = { |
| "bigcode/starcoderdata": { |
| "configured_weight": 0.92, |
| "post_build_cache_ref_revision": EXPECTED_SOURCE_REVISIONS[ |
| "bigcode/starcoderdata" |
| ], |
| "current_main_revision_at_audit": EXPECTED_SOURCE_REVISIONS[ |
| "bigcode/starcoderdata" |
| ], |
| "dataset_card": { |
| "url": ( |
| "https://huggingface.co/datasets/bigcode/starcoderdata/blob/" |
| "9fc30b578cedaec69e47302df72cf00feed7c8c4/README.md" |
| ), |
| "sha256": ( |
| "7a3e42cc82fb48b6b81f2ef06eab94af33e605eff743c6a4b8a3b1852ced7c0a" |
| ), |
| "license_label": "other", |
| "terms": ( |
| "Original repository licenses apply, including attribution " |
| "clauses when relevant. Users must follow the source dataset " |
| "update and removal terms." |
| ), |
| }, |
| "observed_stream_row_schema": { |
| "subset": "python", |
| "fields": [ |
| "content", |
| "id", |
| "max_stars_count", |
| "max_stars_repo_name", |
| "max_stars_repo_path", |
| ], |
| "canonical_fields_sha256": ( |
| "ddfa03121c2f5e5766eada883df62dcaef2a04540a49d849692831ae81fbddd4" |
| ), |
| }, |
| }, |
| "HuggingFaceFW/fineweb-edu": { |
| "configured_weight": 0.08, |
| "post_build_cache_ref_revision": EXPECTED_SOURCE_REVISIONS[ |
| "HuggingFaceFW/fineweb-edu" |
| ], |
| "current_main_revision_at_audit": EXPECTED_SOURCE_REVISIONS[ |
| "HuggingFaceFW/fineweb-edu" |
| ], |
| "dataset_card": { |
| "url": ( |
| "https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu/" |
| "blob/87f09149ef4734204d70ed1d046ddc9ca3f2b8f9/README.md" |
| ), |
| "sha256": ( |
| "a0cc8998a20499432b28b6575f3046b714938eb8e11b8d59a1d25ddf3716061e" |
| ), |
| "license_label": "odc-by", |
| "terms": ( |
| "The dataset is distributed under ODC-By 1.0 and remains " |
| "subject to Common Crawl terms." |
| ), |
| }, |
| "observed_stream_row_schema": { |
| "subset": "sample-10BT", |
| "fields": [ |
| "dump", |
| "file_path", |
| "id", |
| "int_score", |
| "language", |
| "language_score", |
| "score", |
| "text", |
| "token_count", |
| "url", |
| ], |
| "canonical_fields_sha256": ( |
| "a7b0323d3e758514f936736e75a919bda456e98164299c1c1ce5970f65678f91" |
| ), |
| }, |
| }, |
| } |
| EXPECTED_RUN1_GATE = { |
| "structural_filters_applied": True, |
| "extension_parser_activated_for_hub_rows": False, |
| "configured_path_field": None, |
| "path_field_used": "path", |
| "starcoderdata_path_field": "max_stars_repo_path", |
| "reason": ( |
| "The run 1 iterator requested the absent path column, so Hub rows " |
| "reached the structural filters without a file extension. The Python " |
| "ast.parse and JSON json.loads branches therefore did not activate." |
| ), |
| "corpus_script_at_build_sha256": ( |
| "7180e0d69a543fa2ddcf76ef6fa035a14bab2f7e0dfc8a31413b416ad891886e" |
| ), |
| } |
| EXPECTED_CORRECTION_BEHAVIOR = ( |
| "Known Hub schemas select their real path column and fail closed if it is " |
| "missing or empty. A leading StarCoderData reponame metadata line is " |
| "removed only for syntax parsing, while the original text remains the " |
| "training payload." |
| ) |
| EXPECTED_TOKENIZER_SAMPLING = { |
| "documents_requested": 400000, |
| "strategy": "round_robin_by_configured_source_entry", |
| "configured_token_weights_applied": False, |
| "source_entries": 11, |
| "exact_row_manifest_preserved": False, |
| "build_log": { |
| "path": "evidence/tokenizer_build.log", |
| "sha256": ( |
| "7c28f91dc527e0cc37d23c520ba183aef845f948b64ea822f3b3ca17de264467" |
| ), |
| }, |
| "tokenizer": { |
| "path": "tokenizer/code32k.json", |
| "sha256": ( |
| "401a28c1f079050c48f6438830ca772d161d897e3cf2f30588d9ddc587dc6081" |
| ), |
| }, |
| "statement": ( |
| "The tokenizer sample included FineWeb-Edu and sampled source entries " |
| "evenly by document, not according to the later training-token weights." |
| ), |
| } |
| EXPECTED_FINAL_BUILD_EVIDENCE = { |
| "log": { |
| "path": "evidence/run1_corpus_build.log", |
| "sha256": ( |
| "ef5de5c46aac1ff601158b43a3cde481ae6090ea04eefc772c531ff2ac78295e" |
| ), |
| }, |
| "final_index": { |
| "path": "data/shards/index.json", |
| "sha256": ( |
| "862b1a9b7cc6c3c0d31299e21b352e2b736de767a99bf7fa38213d6c60fc0db0" |
| ), |
| }, |
| "realized_train_tokens": { |
| "bigcode/starcoderdata:python": 1200566506, |
| "bigcode/starcoderdata:javascript": 650722190, |
| "bigcode/starcoderdata:typescript": 600701168, |
| "bigcode/starcoderdata:go": 451081764, |
| "bigcode/starcoderdata:rust": 451511705, |
| "bigcode/starcoderdata:java": 400525337, |
| "bigcode/starcoderdata:c": 250363883, |
| "bigcode/starcoderdata:shell": 150445965, |
| "bigcode/starcoderdata:sql": 100765073, |
| "bigcode/starcoderdata:markdown": 351026251, |
| "HuggingFaceFW/fineweb-edu:sample-10BT": 400290897, |
| }, |
| "total_train_tokens": 5008000739, |
| "realized_train_percent": { |
| "implementation_code": 84.997663, |
| "starcoderdata_including_markdown": 92.006972, |
| "markdown": 7.009309, |
| "fineweb_edu": 7.993028, |
| }, |
| "scope": ( |
| "Exact aggregate train-token totals only. Row identities, rejection " |
| "counts, per-source validation overshoot, and row-level obligations " |
| "remain unavailable." |
| ), |
| } |
| EXPECTED_RUN1_FIM_APPLICATION = { |
| "selection_unit": "tokenized_chunk", |
| "maximum_chunk_tokens": 1024, |
| "configured_transform_probability_per_chunk": 0.7, |
| "selected_orderings": { |
| "psm_probability": 0.5, |
| "spm_probability": 0.5, |
| }, |
| "training_window_tokens": 2051, |
| "configured_rate_is_per_source_document": False, |
| "configured_rate_is_per_training_window": False, |
| "build_source": { |
| "git_commit": ( |
| "a534de4d542167bdcea8adfda8fbf25d6cd0db44" |
| ), |
| "path": "scripts/prepare_data.py", |
| "git_blob_sha1": "18b7e158ecec3467be28e1b18a5bab72c0ee1c77", |
| "sha256": ( |
| "6ebbd49a92de87582c429e2c0a5e2fd22792b7db1cbf44e37651b4eceaef7ff6" |
| ), |
| }, |
| "build_log": { |
| "path": "evidence/run1_corpus_build.log", |
| "sha256": ( |
| "ef5de5c46aac1ff601158b43a3cde481ae6090ea04eefc772c531ff2ac78295e" |
| ), |
| }, |
| "statement": ( |
| "Run 1 split each tokenized source document into chunks of at most " |
| "1024 tokens and selected FIM independently for each chunk. The " |
| "configured 0.7 is not a per-document or per-window rate." |
| ), |
| } |
| EXPECTED_RUN1_INTEGRITY = { |
| "source_files": 52, |
| "source_bytes": 10056013702, |
| "source_index_sha256": ( |
| "862b1a9b7cc6c3c0d31299e21b352e2b736de767a99bf7fa38213d6c60fc0db0" |
| ), |
| "attestation_kind": "post_build_current_bytes_and_visible_grammar", |
| "boundary_recovery": { |
| "mode": "deterministic_visible_grammar_normalization", |
| "detectable_reassembly_groups": 27, |
| "restored_internal_eos_tokens": 33, |
| "exact_original_units_proven": False, |
| }, |
| "normalized_splits": { |
| "train": { |
| "source_tokens": 5008000739, |
| "derived_tokens": 4992043184, |
| "units": 7629643, |
| "fim_units": 5319185, |
| }, |
| "val": { |
| "source_tokens": 20006112, |
| "derived_tokens": 19949502, |
| "units": 27087, |
| "fim_units": 18870, |
| }, |
| }, |
| "scheduled_run2_training_positions": 4999872512, |
| } |
| EXPECTED_RUN2_BUILD_CONTRACT = { |
| "schema_version": 2, |
| "strategy": "run1_deterministic_no_fim_normalization_v1", |
| "source_index": { |
| "path": "data/shards/index.json", |
| "sha256": EXPECTED_RUN1_INTEGRITY["source_index_sha256"], |
| "schema_version": 1, |
| }, |
| "source_integrity_receipt": ( |
| "config/run1_shard_integrity_receipt.json" |
| ), |
| "source_integrity_receipt_sha256": ( |
| "5831ecd4a471fbe07e19b212bc3de44bed0b0b6b456083e888f66802937bf471" |
| ), |
| "require_fresh_output_dir": True, |
| } |
| EXPECTED_LIMITATIONS = { |
| "source_row_metadata_preserved_in_shards": False, |
| "per_row_license_mapping_preserved": False, |
| "per_row_attribution_index_available": False, |
| "realized_per_source_train_tokens_recovered_from_final_log": True, |
| "per_source_validation_counts_recorded": False, |
| "locally_verified_permissive_only": False, |
| "source_revisions_captured_during_build": False, |
| "exact_original_unit_boundaries_proven": False, |
| "statement": ( |
| "Run 1 preserves configured source weights, exact aggregate " |
| "train-token totals from the recovered final build log, post-build " |
| "revision evidence, and a post-build hash of every current shard. It " |
| "does not preserve ordered raw rows, repository paths, rejection " |
| "counts, per-source validation overshoot, per-row licenses, attribution " |
| "mapping, or exact original unit boundaries needed for a local " |
| "permissive-only and example-exact audit." |
| ), |
| } |
| PROHIBITED_PUBLICATION_TEXT = ( |
| "92 percent permissively licensed", |
| "permissive-only by construction upstream", |
| "70 percent of its pretraining documents", |
| "FIM is applied per document at prepare time", |
| "to 70 percent of documents", |
| ( |
| "Every document passed a quality gate before tokenization: Python had " |
| "to survive" |
| ), |
| ) |
| EXPECTED_DISCLOSURES = [ |
| ( |
| "Do not claim that every Python or JSON training document passed an " |
| "extension parser." |
| ), |
| ( |
| "Do not claim that the run 1 shards were locally verified as " |
| "permissive-only." |
| ), |
| ( |
| "State that original StarCoderData repository terms and relevant " |
| "attribution clauses still apply." |
| ), |
| ( |
| "State that FineWeb-Edu is ODC-By 1.0 and remains subject to Common " |
| "Crawl terms." |
| ), |
| ( |
| "State that Apache 2.0 covers the Wisp artifact and does not override " |
| "source-data or generated-code terms." |
| ), |
| ( |
| "State that the tokenizer sampled source entries round-robin by " |
| "document rather than using the configured training-token weights." |
| ), |
| ( |
| "Do not claim that deterministic run 1 token normalization proves " |
| "exact original examples or boundaries." |
| ), |
| ( |
| "State that run 1 applied fim_rate 0.7 independently per tokenized " |
| "chunk, not per source document or sampled training window." |
| ), |
| ] |
|
|
|
|
| def file_sha256(path): |
| digest = hashlib.sha256() |
| with open(path, "rb") as f: |
| for block in iter(lambda: f.read(1024 * 1024), b""): |
| digest.update(block) |
| return digest.hexdigest() |
|
|
|
|
| def load_json(path): |
| with open(path, encoding="utf-8") as f: |
| value = json.load(f) |
| if not isinstance(value, dict): |
| raise ValueError(f"{path}: top-level JSON must be an object") |
| return value |
|
|
|
|
| def canonical_fields_sha256(fields): |
| payload = json.dumps( |
| sorted(fields), |
| separators=(",", ":"), |
| ensure_ascii=True, |
| ).encode() |
| return hashlib.sha256(payload).hexdigest() |
|
|
|
|
| def _require(condition, message): |
| if not condition: |
| raise ValueError(message) |
|
|
|
|
| def _validate_artifact(artifact, label): |
| path = artifact.get("path") |
| expected = artifact.get("sha256") |
| _require( |
| isinstance(path, str) and isinstance(expected, str), |
| f"training-data {label} artifact is missing", |
| ) |
| _require( |
| file_sha256(path) == expected, |
| f"training-data {label} hash differs from receipt", |
| ) |
|
|
|
|
| def validate_publication_text(data_text, model_card_text): |
| data_text = " ".join(data_text.split()) |
| model_card_text = " ".join(model_card_text.split()) |
| combined = data_text + "\n" + model_card_text |
| for claim in PROHIBITED_PUBLICATION_TEXT: |
| _require( |
| claim not in combined, |
| f"prohibited training-data claim remains: {claim}", |
| ) |
| required_data = ( |
| "The final build log survived and is now preserved byte-for-byte", |
| "The realized train split is 92.006972 percent StarCoderData", |
| "neither extension parser activated during the run 1 Hub build", |
| "cannot support a local per-file licensing or attribution audit", |
| "trained on 400,000 documents drawn round-robin", |
| "not proof of exact original examples", |
| "FIM is selected independently for each chunk at prepare time", |
| "It was not applied once per source document", |
| ) |
| required_model_card = ( |
| "This is a source percentage, not a permissive-license percentage", |
| "did not activate for run 1", |
| "does not override training-source terms", |
| "sampled round-robin across the eleven source entries", |
| "cannot prove exact original example boundaries", |
| "Exact aggregate train-token totals survive", |
| "70 percent of those chunks were independently transformed", |
| "This is not a per-document or per-window rate", |
| ) |
| for text in required_data: |
| _require(text in data_text, f"DATA.md disclosure is missing: {text}") |
| for text in required_model_card: |
| _require( |
| text in model_card_text, |
| f"MODEL_CARD.md disclosure is missing: {text}", |
| ) |
|
|
|
|
| def _configured_repo_weights(config): |
| weights = {} |
| for source in config.get("sources", []): |
| repo = source.get("repo") |
| _require(isinstance(repo, str), "training source repo is missing") |
| weights[repo] = weights.get(repo, 0.0) + float(source["weight"]) |
| return {repo: round(weight, 12) for repo, weight in weights.items()} |
|
|
|
|
| def validate_training_data_receipt(receipt, receipt_path=None): |
| _require( |
| receipt.get("schema_version") == 1, |
| "training-data receipt schema is not 1", |
| ) |
| _require( |
| receipt.get("status") == "limitations_registered", |
| "training-data limitations are not registered", |
| ) |
|
|
| run1_artifact = receipt.get("registered_run1_config", {}) |
| run2_artifact = receipt.get("registered_run2_config", {}) |
| _validate_artifact(run1_artifact, "run 1 config") |
| _validate_artifact(run2_artifact, "run 2 config") |
| run1 = load_json(run1_artifact["path"]) |
| run2 = load_json(run2_artifact["path"]) |
|
|
| run1_weights = _configured_repo_weights(run1) |
| _require( |
| "sources" not in run2, |
| "derived run 2 config contains executable training sources", |
| ) |
|
|
| sources = receipt.get("sources") |
| _require( |
| sources == EXPECTED_SOURCES, |
| "training-data source evidence differs", |
| ) |
| receipt_weights = { |
| repo: float(source.get("configured_weight")) |
| for repo, source in sources.items() |
| } |
| _require( |
| receipt_weights == run1_weights, |
| "training-data source weights differ from registered configs", |
| ) |
| for repo, revision in EXPECTED_SOURCE_REVISIONS.items(): |
| source = sources[repo] |
| _require( |
| source.get("post_build_cache_ref_revision") == revision |
| and source.get("current_main_revision_at_audit") == revision, |
| f"training-data {repo} revision evidence differs", |
| ) |
| schema = source.get("observed_stream_row_schema", {}) |
| _require( |
| canonical_fields_sha256(schema.get("fields", [])) |
| == schema.get("canonical_fields_sha256"), |
| f"training-data {repo} row schema hash differs", |
| ) |
| run1_gate = receipt.get("run1_quality_gate", {}) |
| _require( |
| run1_gate == EXPECTED_RUN1_GATE, |
| "run 1 quality-gate limitation differs", |
| ) |
|
|
| correction = receipt.get("post_run1_correction", {}) |
| _require( |
| correction.get("effective_scope") == "future_source_streaming_only" |
| and correction.get("active_run1_process_or_shards_changed") is False |
| and correction.get("behavior") == EXPECTED_CORRECTION_BEHAVIOR, |
| "post-run 1 correction scope differs", |
| ) |
| _validate_artifact(correction.get("corpus_script", {}), "corpus script") |
| _validate_artifact( |
| correction.get("quality_gate_test", {}), |
| "quality-gate test", |
| ) |
|
|
| _require( |
| receipt.get("tokenizer_sampling") == EXPECTED_TOKENIZER_SAMPLING, |
| "tokenizer sampling limitation differs", |
| ) |
| _validate_artifact( |
| receipt["tokenizer_sampling"]["build_log"], |
| "tokenizer build log", |
| ) |
| _validate_artifact( |
| receipt["tokenizer_sampling"]["tokenizer"], |
| "tokenizer", |
| ) |
|
|
| build_evidence = receipt.get("run1_final_build_evidence") |
| _require( |
| build_evidence == EXPECTED_FINAL_BUILD_EVIDENCE, |
| "run 1 final-build evidence differs", |
| ) |
| _validate_artifact(build_evidence["log"], "run 1 final build log") |
| _validate_artifact(build_evidence["final_index"], "run 1 final index") |
| _require( |
| sum(build_evidence["realized_train_tokens"].values()) |
| == build_evidence["total_train_tokens"], |
| "run 1 realized train-token totals do not sum", |
| ) |
|
|
| fim_application = receipt.get("run1_fim_application") |
| _require( |
| fim_application == EXPECTED_RUN1_FIM_APPLICATION, |
| "run 1 FIM application evidence differs", |
| ) |
| _validate_artifact( |
| fim_application["build_log"], |
| "run 1 FIM build log", |
| ) |
|
|
| integrity = receipt.get("run1_shard_integrity", {}) |
| integrity_artifact = integrity.get("receipt", {}) |
| _validate_artifact(integrity_artifact, "run 1 shard integrity receipt") |
| _require( |
| { |
| key: value |
| for key, value in integrity.items() |
| if key != "receipt" |
| } |
| == EXPECTED_RUN1_INTEGRITY, |
| "run 1 shard integrity summary differs", |
| ) |
| integrity_receipt = load_json(integrity_artifact["path"]) |
| _require( |
| integrity_receipt.get("schema_version") == 1 |
| and integrity_receipt.get("status") == "complete" |
| and integrity_receipt.get("algorithm") |
| == "run1_deterministic_no_fim_normalization_v1", |
| "run 1 shard integrity receipt contract differs", |
| ) |
| _require( |
| integrity_receipt.get("source_index", {}).get("sha256") |
| == EXPECTED_RUN1_INTEGRITY["source_index_sha256"], |
| "run 1 source index hash differs", |
| ) |
| _require( |
| { |
| key: integrity_receipt.get("boundary_recovery", {}).get(key) |
| for key in EXPECTED_RUN1_INTEGRITY["boundary_recovery"] |
| } |
| == EXPECTED_RUN1_INTEGRITY["boundary_recovery"], |
| "run 1 boundary-recovery evidence differs", |
| ) |
| for split, expected in EXPECTED_RUN1_INTEGRITY[ |
| "normalized_splits" |
| ].items(): |
| actual = integrity_receipt.get("splits", {}).get(split, {}) |
| summary = { |
| "source_tokens": actual.get("source_tokens"), |
| "derived_tokens": actual.get("derived_tokens"), |
| "units": actual.get("units"), |
| "fim_units": ( |
| actual.get("fim_psm_units", 0) |
| + actual.get("fim_spm_units", 0) |
| ), |
| } |
| _require( |
| summary == expected, |
| f"run 1 normalized {split} evidence differs", |
| ) |
|
|
| expected_integrity_reference = { |
| "role": "current", |
| "receipt": integrity_artifact["path"], |
| "sha256": integrity_artifact["sha256"], |
| } |
| _require( |
| run1.get("data_integrity") == expected_integrity_reference, |
| "run 1 data-integrity config differs", |
| ) |
| expected_integrity_reference["role"] = "source" |
| _require( |
| run2.get("data_integrity") == expected_integrity_reference, |
| "run 2 source-integrity config differs", |
| ) |
| _require( |
| run2.get("data_build_contract") == EXPECTED_RUN2_BUILD_CONTRACT, |
| "run 2 deterministic normalization contract differs", |
| ) |
|
|
| documents = receipt.get("publication_documents", {}) |
| data_artifact = documents.get("data_document", {}) |
| model_card_artifact = documents.get("model_card_template", {}) |
| _validate_artifact(data_artifact, "data document") |
| _validate_artifact(model_card_artifact, "model-card template") |
| with open(data_artifact["path"], encoding="utf-8") as f: |
| data_text = f.read() |
| with open(model_card_artifact["path"], encoding="utf-8") as f: |
| model_card_text = f.read() |
| validate_publication_text(data_text, model_card_text) |
|
|
| limitations = receipt.get("run1_provenance_limitations", {}) |
| _require( |
| limitations == EXPECTED_LIMITATIONS, |
| "run 1 provenance limitations differ", |
| ) |
| _require( |
| receipt.get("required_publication_disclosures") |
| == EXPECTED_DISCLOSURES, |
| "training-data publication disclosures differ", |
| ) |
| return { |
| "receipt_path": receipt_path, |
| "receipt_sha256": ( |
| file_sha256(receipt_path) if receipt_path is not None else None |
| ), |
| "source_weights": receipt_weights, |
| "source_revisions": EXPECTED_SOURCE_REVISIONS, |
| "future_parser_fix_bound": True, |
| "run1_shard_integrity_bound": True, |
| "deterministic_normalization_not_exact_original_units": True, |
| "publication_documents": { |
| key: value["sha256"] |
| for key, value in documents.items() |
| }, |
| } |
|
|
|
|
| def main(): |
| path = os.path.join("config", "training_data_receipt.json") |
| receipt = load_json(path) |
| evidence = validate_training_data_receipt(receipt, path) |
| print( |
| "Training-data limitations and future schema-aware parser fix: PASS " |
| f"({evidence['receipt_sha256']})" |
| ) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
