code/ledger.py

import json, re, shlex, subprocess
from collections import Counter
pg = lambda f: {c['number']: c for c in json.loads(re.search(r'const DATA = (.*);\nconst KEY', open(f).read(), re.S).group(1).replace('<\\/', '</'))}
tri, cl, iv, op, mo = pg('triage.html'), pg('cleanup.html'), pg('issues_v4.html'), pg('open_prs.html'), pg('morning.html')
L = {}  # (kind, n) -> dict
def rec(kind, n, **kw):
    n = int(n); r = L.setdefault((kind, n), dict(kind=kind, n=n, actions=[], closed=False, cat=None, msg=None, title=''))
    a = kw.pop('action', None)
    if a and a not in r['actions']: r['actions'].append(a)
    for k, v in kw.items():
        if v: r[k] = v
    return r
CAT = {'outdated': 'outdated (v4-era, inactive)', 'old_branch': 'old branch / code moved', 'philosophy': 'not aligned with v5',
       'removed': 'removed in v5', 'stale_close': 'stale', 'already_fixed': 'already fixed', 'duplicate_open': 'duplicate', 'fixed': 'fixed'}
cat = lambda c: CAT.get(c, c)
# run 1
toks = shlex.split(open('cmds.sh').read())
for i, t in enumerate(toks):
    if t == 'gh':
        k, n = toks[i + 1], toks[i + 3]; m = toks[toks.index('-c', i) + 1]
        rec(k, n, action='closed', closed=True, cat='fixed' if k == 'issue' else 'superseded', msg=m, title=(tri.get(int(n)) or {}).get('title', ''))
# run 2 cleanup (711)
toks = shlex.split(open('cleanup_close.sh').read().split('R=huggingface/transformers\n', 1)[1]); cur = None; cmsg = {}
for t in toks:
    if t == 'gh': cur = []
    else:
        cur.append(t)
        if len(cur) == 7: cmsg[int(cur[2])] = cur[6]
V4 = next(c['message'] for c in cl.values() if 'based on v4' in c['message'])
for n in set(int(x) for x in re.findall(r'#(\d+) ok', open('cleanup_run.log').read() + open('cleanup_run2.log').read())):
    m = V4 if n in {13752, 15773, 18152} else cmsg.get(n, cl[n]['message'])
    rec('pr', n, action='closed', closed=True, cat=cat(cl[n]['category']), msg=m, title=cl[n]['title'])
# issue run (first 80)
jobs = {j['n']: j for j in json.load(open('issues_jobs.json'))}
for n in set(int(x) for x in re.findall(r'#(\d+) \w+ ok', open('issues_run.log').read())):
    rec('issue', n, action='closed' if jobs[n]['op'] == 'close' else 'commented', closed=jobs[n]['op'] == 'close', cat=cat(iv[n]['category']), msg=jobs[n]['msg'], title=iv[n]['title'])
def okset(log): return set((k, int(n), o) for k, n, o in re.findall(r'(issue|pr)#(\d+) (\w+)(?: \S+)? ok', open(log).read()))
# batch A/B
ok = okset('final_run.log')
for q in json.load(open('queue_A.json')) + json.load(open('queue_B.json')):
    if (q['kind'], q['n'], q['op']) not in ok: continue
    src = iv if q['kind'] == 'issue' else op; c = src.get(q['n'], {})
    if q['op'] == 'comment': rec('issue', q['n'], action='commented', cat={'new_model': 'new-model details', 'pointers': 'pointers (how to do it)'}.get(c.get('category'), c.get('category')), msg=q['msg'], title=c.get('title', ''))
    else: rec(q['kind'], q['n'], action='closed', closed=True, cat=cat(c.get('category', 'stale')), msg=q['msg'], title=c.get('title', ''))
# morning
ok = okset('morning_run.log')
for q in json.load(open('queue_M.json')):
    if (q['kind'], q['n'], q['op']) not in ok: continue
    c = mo.get(q['n'], {}); src = (iv if q['kind'] == 'issue' else op).get(q['n'], {})
    if q['op'] == 'edit': rec(q['kind'], q['n'], action='comment corrected', msg=q['msg'])
    else: rec(q['kind'], q['n'], action='closed', closed=True, cat=cat(src.get('category')) if src.get('category') in ('fixed', 'superseded', 'obsolete', 'removed') else 'stale', msg=q['msg'], title=c.get('title') or src.get('title', ''))
# nudge-audit run (R) + final queue (F)
NV = {r['number']: r for k in range(3) for r in json.load(open(f'results_N{k}.json'))}
QA = {c['number']: c for c in json.load(open('qa_cards.json'))}
for f, log in (('queue_R.json', 'reviews_run.log'), ('queue_F.json', 'final2_run.log')):
    ok = okset(log)
    for q in json.load(open(f)):
        if (q['kind'], q['n'], q['op']) not in ok: continue
        t = (NV.get(q['n']) or QA.get(q['n']) or {}).get('title', '')
        if q['op'] == 'addrev': rec(q['kind'], q['n'], action=f"review requested: @{q['reviewer']}", title=t)
        elif q['op'] == 'comment': rec(q['kind'], q['n'], action='maintainer pinged' if q['msg'].startswith('@') else 'commented', msg=q['msg'], title=t)
        else:
            v = (QA.get(q['n']) or {}).get('category') if f == 'queue_F.json' else (NV.get(q['n']) or {}).get('verdict')
            rec(q['kind'], q['n'], action='closed', closed=True, cat=cat(v or 'superseded'), msg=q['msg'], title=t)
# labels
for line in open('labels.txt'):
    p = line.split(); lab = ' '.join(x for x in p if not x.isdigit())
    for n in p:
        if n.isdigit(): rec('issue', n, action=f'labelled: {lab}')
# exporters edits
for p in json.load(open('exporters_plan.json')):
    k = 'pr' if p['n'] == 29315 else 'issue'
    rec(k, p['n'], action='comment corrected', msg=p['new'])
# open items (fresh snapshot)
open_rows = {('pr' if r['k'] == 'PR' else 'issue', r['n']): r for r in json.load(open('land_rows.json'))}
items = []
for key, r in L.items():
    o = open_rows.get(key)
    if r['closed'] and o: r['closed'] = False  # reopened by someone (e.g. #41827)
    if o: r['title'] = r['title'] or o['t']
for key, o in open_rows.items():
    r = L.get(key) or dict(kind=key[0], n=key[1], actions=[], closed=False, cat=None, msg=None, title=o['t'])
    r.update(area=o['area'], type=o['kind'], status=o['status'], author=o['au'], age=o['age'], idle=o['idle'], models=', '.join(o['models']))
    L[key] = r
need = [k for k, r in L.items() if not r['title']]
for i in range(0, len(need), 80):
    part = need[i:i + 80]
    q = '{repository(owner:"huggingface",name:"transformers"){' + ' '.join(f'n{n}: issueOrPullRequest(number:{n}){{... on Issue{{title}} ... on PullRequest{{title}}}}' for _, n in part) + '}}'
    d = json.loads(subprocess.run(['gh', 'api', 'graphql', '-f', 'query=' + q], capture_output=True, text=True).stdout)['data']['repository']
    for k in part: L[k]['title'] = (d.get(f'n{k[1]}') or {}).get('title', '')
for (k, n), r in sorted(L.items(), key=lambda x: -x[0][1]):
    isopen = (k, n) in open_rows
    if r['closed'] and not isopen: group = 'Closed issues' if k == 'issue' else 'Closed PRs'; category = r['cat'] or 'other'
    elif isopen: group = 'Open issues' if k == 'issue' else 'Open PRs'; category = (r.get('type') if k == 'issue' else r.get('status')) or 'other'
    else: group = 'Commented (now closed by others)'; category = r['cat'] or 'other'
    items.append(dict(key=f'{k}#{n}', n=n, kind=('issue' if k == 'issue' else 'PR'), group=group, category=category, area=r.get('area') or '',
                      type=r.get('type') or '', title=r['title'] or '', action='; '.join(r['actions']) or ('—' if isopen else ''),
                      comment=(r['msg'] or '')[:900], author=r.get('author') or '', idle_days=r.get('idle') if isopen else None, models=r.get('models') or '',
                      url=f"https://github.com/huggingface/transformers/{'pull' if k == 'pr' else 'issues'}/{n}"))
json.dump(items, open('triage_items.json', 'w'))
print(len(items), 'items;', round(len(json.dumps(items)) / 1e6, 2), 'MB')
print(Counter(i['group'] for i in items))
for g in ('Closed issues', 'Closed PRs'): print(g, Counter(i['category'] for i in items if i['group'] == g).most_common())
print('open acted-on:', sum(1 for i in items if i['group'].startswith('Open') and i['action'] not in ('—', '')))