| |
| """Standalone tokenizer / vocab alignment check for cross-model OPD.""" |
| from __future__ import annotations |
|
|
| import argparse |
| import os |
| import sys |
|
|
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
|
|
| from transformers import AutoProcessor |
|
|
| from opsd_utils.vocab_align import print_vocab_align_report, verify_shared_tokenizer_alignment |
|
|
|
|
| def main() -> int: |
| parser = argparse.ArgumentParser(description="Verify student/teacher shared vocab alignment") |
| parser.add_argument("--student", required=True, help="Student model id or path") |
| parser.add_argument("--teacher", required=True, help="Teacher model id or path") |
| parser.add_argument("--full-scan", action="store_true", help="Check every token id (slow)") |
| parser.add_argument("--stride", type=int, default=500, help="Sample stride when not full-scan") |
| args = parser.parse_args() |
|
|
| student_proc = AutoProcessor.from_pretrained(args.student) |
| teacher_proc = AutoProcessor.from_pretrained(args.teacher) |
| shared = min(len(student_proc.tokenizer), len(teacher_proc.tokenizer)) |
| report = verify_shared_tokenizer_alignment( |
| student_proc.tokenizer, |
| teacher_proc.tokenizer, |
| shared_vocab=shared, |
| full_scan=args.full_scan, |
| sample_stride=args.stride, |
| ) |
| print_vocab_align_report(report) |
| return 0 if report["aligned"] else 1 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
