File size: 973 Bytes
53f0cc2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | """
Quick verification for reprocess_tokenized_from_clean.py.
"""
from __future__ import annotations
import sys
from pathlib import Path
# Ensure imports work from project root.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from scripts.reprocess_tokenized_from_clean import main as reprocess_main # noqa: E402
if __name__ == "__main__":
try:
sys.argv = [
"verify_reprocess_tokenized_from_clean.py",
"--config",
"configs/component3_reprocess_from_clean.yaml",
"--max_records",
"500",
]
reprocess_main()
print("")
print("Reprocess verification passed.")
except Exception as exc:
print("Reprocess verification failed.")
print(f"What went wrong: {exc}")
print("Fix suggestion: verify input clean file and tokenizer path.")
raise SystemExit(1)
|