assets/scripts/apifid.py

import json, re, sys, glob, os
IDX=json.load(open("/root/27B/docs/fable_review_2026-07-02/verifier_pilot/api_index.json"))
S2F=IDX["short_to_full"]; T=IDX["types"]
def tmembers(short):
    m=set()
    for fn in S2F.get(short,[]): m|=set(T[fn].get("members",[]))
    return m
def exists(typ,mem):
    ms=tmembers(typ)
    if mem in ms: return True
    if (mem.startswith(("get_","set_"))) and mem[4:] in ms: return True
    if ("get_"+mem) in ms or ("set_"+mem) in ms: return True
    return False
MEMBER=re.compile(r'\b([A-Z][A-Za-z0-9]{2,})\.([A-Za-z_][A-Za-z0-9_]*)')
def score(path):
    rows=[json.loads(l) for l in open(path)]
    known=e=m=0; miss_examples=[]
    for r in rows:
        for typ,mem in set(MEMBER.findall(r["generation"])):
            if typ in S2F:
                known+=1
                if exists(typ,mem): e+=1
                else:
                    m+=1
                    if len(miss_examples)<6: miss_examples.append(f"{typ}.{mem}")
    fid=e/(e+m) if (e+m) else None
    return len(rows),known,e,m,fid,miss_examples
print("%-10s %5s %8s %6s %6s %8s"%("ckpt","rows","revitRef","exist","miss","API-fid"))
for f in sorted(glob.glob("/root/27B/grade_out/gen_*.jsonl")):
    n,k,e,m,fid,ex=score(f)
    lab=os.path.basename(f).replace("gen_","").replace(".jsonl","")
    print("%-10s %5d %8d %6d %6d %8s"%(lab,n,k,e,m,("%.1f%%"%(fid*100)) if fid is not None else "n/a"))