code/src/preprocessing/build_mzl_abz_map.py

#!/usr/bin/env python3
"""Build MZL ↔ ABZ mapping from Oracc OSL (osl.asl).

Input: /tmp/osl/00lib/osl.asl (Oracc Global Sign List, CC0)
Output: JSON with:
  - mzl_to_abz: MZL_num → ABZ_num
  - mzl_to_sign: MZL_num → canonical sign name
  - abz_to_sign: ABZ_num → canonical sign name
  - sign_to_mzl: sign → MZL_num
  - sign_to_abz: sign → ABZ_num
"""
import json, re, argparse
from pathlib import Path


def parse_asl(path):
    """Parse @sign ... @end sign blocks from osl.asl."""
    blocks = []
    cur = None
    with open(path) as f:
        for line in f:
            line = line.rstrip()
            if line.startswith('@sign '):
                name = line[6:].strip()
                cur = {'sign': name, 'lists': {}, 'uname': None, 'ucun': None}
            elif line.startswith('@sign- '):
                cur = None  # skip deprecated signs
            elif cur is not None and line.startswith('@list\t'):
                token = line[6:].strip()
                # token format: "MZL001" or "ABZ123" or "ABZ123v" etc
                m = re.match(r'^([A-Z]+)(\S+)$', token)
                if m:
                    lst, num = m.group(1), m.group(2)
                    # Prefer first occurrence per list
                    if lst not in cur['lists']:
                        cur['lists'][lst] = num
            elif cur is not None and line.startswith('@uname\t'):
                cur['uname'] = line[7:].strip()
            elif cur is not None and line.startswith('@ucun\t'):
                cur['ucun'] = line[6:].strip()
            elif line.startswith('@end sign'):
                if cur and cur['sign']:
                    blocks.append(cur)
                cur = None
    return blocks


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--osl', default='/tmp/osl/00lib/osl.asl')
    ap.add_argument('--output', required=True)
    args = ap.parse_args()

    blocks = parse_asl(args.osl)
    print(f"Parsed {len(blocks)} signs")

    mzl_to_abz = {}
    mzl_to_sign = {}
    abz_to_sign = {}
    sign_to_mzl = {}
    sign_to_abz = {}

    for b in blocks:
        sign = b['sign']
        mzl = b['lists'].get('MZL')
        # In Oracc OSL the Borger list appears as ABZL (Assyrisch-Babylonische
        # Zeichenliste); our codebase historically calls it "ABZ", so treat
        # ABZL as the ABZ source.
        abz = b['lists'].get('ABZL') or b['lists'].get('ABZ')
        if mzl:
            # MZL numbers may have leading zeros and optional 'a','b','v' suffixes
            # strip leading zeros of pure-numeric prefix
            mzl_clean = re.sub(r'^0+', '', mzl) or '0'
            mzl_to_sign[mzl] = sign
            mzl_to_sign[mzl_clean] = sign  # also with stripped
            sign_to_mzl[sign] = mzl
            if abz:
                abz_clean = re.sub(r'^0+', '', abz) or '0'
                mzl_to_abz[mzl] = abz
                mzl_to_abz[mzl_clean] = abz
        if abz:
            abz_to_sign[abz] = sign
            abz_to_sign[re.sub(r'^0+', '', abz) or '0'] = sign
            sign_to_abz[sign] = abz

    print(f"MZL entries: {len(sign_to_mzl)}")
    print(f"ABZ entries: {len(sign_to_abz)}")
    print(f"MZL→ABZ mappings: {len(mzl_to_abz)}")

    # Report a few examples
    for mzl in ['MZL001', 'MZL839', 'MZL252', 'MZL090']:
        if mzl in mzl_to_sign:
            print(f"  {mzl} = {mzl_to_sign[mzl]} (ABZ: {mzl_to_abz.get(mzl, '?')})")

    out = {
        'mzl_to_abz': mzl_to_abz,
        'mzl_to_sign': mzl_to_sign,
        'abz_to_sign': abz_to_sign,
        'sign_to_mzl': sign_to_mzl,
        'sign_to_abz': sign_to_abz,
        'n_signs_total': len(blocks),
    }
    Path(args.output).parent.mkdir(parents=True, exist_ok=True)
    with open(args.output, 'w') as f:
        json.dump(out, f, indent=1, ensure_ascii=False)
    print(f"Saved → {args.output}")

if __name__ == '__main__':
    main()