File size: 4,414 Bytes
216004d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | from __future__ import annotations
# llm-wiki-version: 2.0.0
# runtime: ci-safe (skips raw existence check when PROJECT_RAW_ROOT unset)
import csv
import json
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MANIFEST = ROOT / "manifests" / "raw_sources.csv"
META = ROOT / "manifests" / "raw_sources.meta.json"
# Schema versions this script knows how to validate.
KNOWN_SCHEMA_VERSIONS = {1, 2}
LATEST_SCHEMA_VERSION = 2
# Default schema for projects bootstrapped before meta.json existed.
LEGACY_DEFAULT = {
"schema_version": 1,
"columns": [
"source_id",
"company",
"vendor",
"kind",
"filename",
"raw_rel_path",
"status",
"compiled_into",
"notes",
],
"allowed_status": ["new", "compiled", "archived"],
}
def load_schema() -> tuple[dict, str]:
"""Return (schema_dict, source) where source is 'meta' or 'legacy'."""
if META.exists():
try:
data = json.loads(META.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise SystemExit(f"raw_manifest_check: malformed {META.name}: {exc}")
return data, "meta"
return LEGACY_DEFAULT, "legacy"
def main() -> int:
errors: list[str] = []
if not MANIFEST.exists():
print(f"raw_manifest_check: missing {MANIFEST}")
return 1
schema, schema_source = load_schema()
schema_version = schema.get("schema_version", 1)
if schema_version not in KNOWN_SCHEMA_VERSIONS:
if schema_version > LATEST_SCHEMA_VERSION:
print(f"raw_manifest_check: SKIPPED")
print(f"- manifest schema_version is {schema_version}, this script knows only up to {LATEST_SCHEMA_VERSION}")
print(f"- run: python3 scripts/version_check.py (or upgrade LLM-wiki to a newer release)")
return 0 # Forward-compat: don't fail the user's CI on a newer manifest.
errors.append(f"unknown schema_version {schema_version} (this script supports {sorted(KNOWN_SCHEMA_VERSIONS)})")
expected_columns = schema["columns"]
allowed_status = set(schema["allowed_status"])
raw_root_env = os.environ.get("PROJECT_RAW_ROOT")
raw_root = Path(raw_root_env).expanduser().resolve() if raw_root_env else None
with MANIFEST.open("r", encoding="utf-8-sig", newline="") as handle:
reader = csv.DictReader(handle)
if reader.fieldnames != expected_columns:
errors.append(f"manifest columns mismatch: expected {expected_columns}, got {reader.fieldnames}")
seen_ids: set[str] = set()
for idx, row in enumerate(reader, start=2):
source_id = (row.get("source_id") or "").strip()
filename = (row.get("filename") or "").strip()
raw_rel_path = (row.get("raw_rel_path") or "").strip()
status = (row.get("status") or "").strip()
compiled_into = (row.get("compiled_into") or "").strip()
if not source_id:
errors.append(f"row {idx}: empty source_id")
elif source_id in seen_ids:
errors.append(f"row {idx}: duplicate source_id {source_id}")
else:
seen_ids.add(source_id)
if not filename:
errors.append(f"row {idx}: empty filename")
if not raw_rel_path:
errors.append(f"row {idx}: empty raw_rel_path")
if status not in allowed_status:
errors.append(f"row {idx}: bad status {status}")
if raw_root and raw_rel_path:
candidate = (raw_root / raw_rel_path).resolve()
if not candidate.exists():
errors.append(f"row {idx}: missing local raw file {candidate}")
if status == "compiled" and not compiled_into:
errors.append(f"row {idx}: status {status} requires compiled_into")
if errors:
print("raw_manifest_check: FAILED")
for item in errors:
print(f"- {item}")
return 1
print("raw_manifest_check: OK")
print(f"- manifest: {MANIFEST}")
print(f"- schema: v{schema_version} (loaded from {schema_source})")
if raw_root:
print(f"- PROJECT_RAW_ROOT: {raw_root}")
else:
print("- PROJECT_RAW_ROOT: not set, existence checks skipped")
return 0
if __name__ == "__main__":
sys.exit(main())
|