Spaces:
Running
Running
| import json, re, shlex, subprocess | |
| from collections import Counter | |
| pg = lambda f: {c['number']: c for c in json.loads(re.search(r'const DATA = (.*);\nconst KEY', open(f).read(), re.S).group(1).replace('<\\/', '</'))} | |
| tri, cl, iv, op, mo = pg('triage.html'), pg('cleanup.html'), pg('issues_v4.html'), pg('open_prs.html'), pg('morning.html') | |
| L = {} # (kind, n) -> dict | |
| def rec(kind, n, **kw): | |
| n = int(n); r = L.setdefault((kind, n), dict(kind=kind, n=n, actions=[], closed=False, cat=None, msg=None, title='')) | |
| a = kw.pop('action', None) | |
| if a and a not in r['actions']: r['actions'].append(a) | |
| for k, v in kw.items(): | |
| if v: r[k] = v | |
| return r | |
| CAT = {'outdated': 'outdated (v4-era, inactive)', 'old_branch': 'old branch / code moved', 'philosophy': 'not aligned with v5', | |
| 'removed': 'removed in v5', 'stale_close': 'stale', 'already_fixed': 'already fixed', 'duplicate_open': 'duplicate', 'fixed': 'fixed'} | |
| cat = lambda c: CAT.get(c, c) | |
| # run 1 | |
| toks = shlex.split(open('cmds.sh').read()) | |
| for i, t in enumerate(toks): | |
| if t == 'gh': | |
| k, n = toks[i + 1], toks[i + 3]; m = toks[toks.index('-c', i) + 1] | |
| rec(k, n, action='closed', closed=True, cat='fixed' if k == 'issue' else 'superseded', msg=m, title=(tri.get(int(n)) or {}).get('title', '')) | |
| # run 2 cleanup (711) | |
| toks = shlex.split(open('cleanup_close.sh').read().split('R=huggingface/transformers\n', 1)[1]); cur = None; cmsg = {} | |
| for t in toks: | |
| if t == 'gh': cur = [] | |
| else: | |
| cur.append(t) | |
| if len(cur) == 7: cmsg[int(cur[2])] = cur[6] | |
| V4 = next(c['message'] for c in cl.values() if 'based on v4' in c['message']) | |
| for n in set(int(x) for x in re.findall(r'#(\d+) ok', open('cleanup_run.log').read() + open('cleanup_run2.log').read())): | |
| m = V4 if n in {13752, 15773, 18152} else cmsg.get(n, cl[n]['message']) | |
| rec('pr', n, action='closed', closed=True, cat=cat(cl[n]['category']), msg=m, title=cl[n]['title']) | |
| # issue run (first 80) | |
| jobs = {j['n']: j for j in json.load(open('issues_jobs.json'))} | |
| for n in set(int(x) for x in re.findall(r'#(\d+) \w+ ok', open('issues_run.log').read())): | |
| rec('issue', n, action='closed' if jobs[n]['op'] == 'close' else 'commented', closed=jobs[n]['op'] == 'close', cat=cat(iv[n]['category']), msg=jobs[n]['msg'], title=iv[n]['title']) | |
| def okset(log): return set((k, int(n), o) for k, n, o in re.findall(r'(issue|pr)#(\d+) (\w+)(?: \S+)? ok', open(log).read())) | |
| # batch A/B | |
| ok = okset('final_run.log') | |
| for q in json.load(open('queue_A.json')) + json.load(open('queue_B.json')): | |
| if (q['kind'], q['n'], q['op']) not in ok: continue | |
| src = iv if q['kind'] == 'issue' else op; c = src.get(q['n'], {}) | |
| if q['op'] == 'comment': rec('issue', q['n'], action='commented', cat={'new_model': 'new-model details', 'pointers': 'pointers (how to do it)'}.get(c.get('category'), c.get('category')), msg=q['msg'], title=c.get('title', '')) | |
| else: rec(q['kind'], q['n'], action='closed', closed=True, cat=cat(c.get('category', 'stale')), msg=q['msg'], title=c.get('title', '')) | |
| # morning | |
| ok = okset('morning_run.log') | |
| for q in json.load(open('queue_M.json')): | |
| if (q['kind'], q['n'], q['op']) not in ok: continue | |
| c = mo.get(q['n'], {}); src = (iv if q['kind'] == 'issue' else op).get(q['n'], {}) | |
| if q['op'] == 'edit': rec(q['kind'], q['n'], action='comment corrected', msg=q['msg']) | |
| else: rec(q['kind'], q['n'], action='closed', closed=True, cat=cat(src.get('category')) if src.get('category') in ('fixed', 'superseded', 'obsolete', 'removed') else 'stale', msg=q['msg'], title=c.get('title') or src.get('title', '')) | |
| # nudge-audit run (R) + final queue (F) | |
| NV = {r['number']: r for k in range(3) for r in json.load(open(f'results_N{k}.json'))} | |
| QA = {c['number']: c for c in json.load(open('qa_cards.json'))} | |
| for f, log in (('queue_R.json', 'reviews_run.log'), ('queue_F.json', 'final2_run.log')): | |
| ok = okset(log) | |
| for q in json.load(open(f)): | |
| if (q['kind'], q['n'], q['op']) not in ok: continue | |
| t = (NV.get(q['n']) or QA.get(q['n']) or {}).get('title', '') | |
| if q['op'] == 'addrev': rec(q['kind'], q['n'], action=f"review requested: @{q['reviewer']}", title=t) | |
| elif q['op'] == 'comment': rec(q['kind'], q['n'], action='maintainer pinged' if q['msg'].startswith('@') else 'commented', msg=q['msg'], title=t) | |
| else: | |
| v = (QA.get(q['n']) or {}).get('category') if f == 'queue_F.json' else (NV.get(q['n']) or {}).get('verdict') | |
| rec(q['kind'], q['n'], action='closed', closed=True, cat=cat(v or 'superseded'), msg=q['msg'], title=t) | |
| # labels | |
| for line in open('labels.txt'): | |
| p = line.split(); lab = ' '.join(x for x in p if not x.isdigit()) | |
| for n in p: | |
| if n.isdigit(): rec('issue', n, action=f'labelled: {lab}') | |
| # exporters edits | |
| for p in json.load(open('exporters_plan.json')): | |
| k = 'pr' if p['n'] == 29315 else 'issue' | |
| rec(k, p['n'], action='comment corrected', msg=p['new']) | |
| # open items (fresh snapshot) | |
| open_rows = {('pr' if r['k'] == 'PR' else 'issue', r['n']): r for r in json.load(open('land_rows.json'))} | |
| items = [] | |
| for key, r in L.items(): | |
| o = open_rows.get(key) | |
| if r['closed'] and o: r['closed'] = False # reopened by someone (e.g. #41827) | |
| if o: r['title'] = r['title'] or o['t'] | |
| for key, o in open_rows.items(): | |
| r = L.get(key) or dict(kind=key[0], n=key[1], actions=[], closed=False, cat=None, msg=None, title=o['t']) | |
| r.update(area=o['area'], type=o['kind'], status=o['status'], author=o['au'], age=o['age'], idle=o['idle'], models=', '.join(o['models'])) | |
| L[key] = r | |
| need = [k for k, r in L.items() if not r['title']] | |
| for i in range(0, len(need), 80): | |
| part = need[i:i + 80] | |
| q = '{repository(owner:"huggingface",name:"transformers"){' + ' '.join(f'n{n}: issueOrPullRequest(number:{n}){{... on Issue{{title}} ... on PullRequest{{title}}}}' for _, n in part) + '}}' | |
| d = json.loads(subprocess.run(['gh', 'api', 'graphql', '-f', 'query=' + q], capture_output=True, text=True).stdout)['data']['repository'] | |
| for k in part: L[k]['title'] = (d.get(f'n{k[1]}') or {}).get('title', '') | |
| for (k, n), r in sorted(L.items(), key=lambda x: -x[0][1]): | |
| isopen = (k, n) in open_rows | |
| if r['closed'] and not isopen: group = 'Closed issues' if k == 'issue' else 'Closed PRs'; category = r['cat'] or 'other' | |
| elif isopen: group = 'Open issues' if k == 'issue' else 'Open PRs'; category = (r.get('type') if k == 'issue' else r.get('status')) or 'other' | |
| else: group = 'Commented (now closed by others)'; category = r['cat'] or 'other' | |
| items.append(dict(key=f'{k}#{n}', n=n, kind=('issue' if k == 'issue' else 'PR'), group=group, category=category, area=r.get('area') or '', | |
| type=r.get('type') or '', title=r['title'] or '', action='; '.join(r['actions']) or ('—' if isopen else ''), | |
| comment=(r['msg'] or '')[:900], author=r.get('author') or '', idle_days=r.get('idle') if isopen else None, models=r.get('models') or '', | |
| url=f"https://github.com/huggingface/transformers/{'pull' if k == 'pr' else 'issues'}/{n}")) | |
| json.dump(items, open('triage_items.json', 'w')) | |
| print(len(items), 'items;', round(len(json.dumps(items)) / 1e6, 2), 'MB') | |
| print(Counter(i['group'] for i in items)) | |
| for g in ('Closed issues', 'Closed PRs'): print(g, Counter(i['category'] for i in items if i['group'] == g).most_common()) | |
| print('open acted-on:', sum(1 for i in items if i['group'].startswith('Open') and i['action'] not in ('—', ''))) | |