code/land_build.py

import json, re, html as H
from collections import Counter
from datetime import datetime, timezone
NOW = datetime(2026, 10, 21, tzinfo=timezone.utc)
age = lambda s: (NOW - datetime.fromisoformat(s.replace('Z', '+00:00'))).days
D = json.load(open('land.json'))
MODELS = set(open('models_main.txt').read().split())
HF = {'MEMBER', 'OWNER', 'COLLABORATOR'}
BOTS = re.compile(r'(?i)bot|dependabot|github-actions|hf-security')
AREA_RULES = [  # (regex on path, area) — first match per file
 (r'^src/transformers/models/[^/]+/(image_processing|video_processing|feature_extraction|processing)_', 'processors (image/video/audio)'),
 (r'^src/transformers/models/', 'models'), (r'^src/transformers/generation/continuous_batching', 'serving / continuous batching'),
 (r'^src/transformers/generation/', 'generation'), (r'^src/transformers/(trainer|training_args|optimization)', 'trainer'),
 (r'tokenization', 'tokenization'), (r'^src/transformers/(quantizers/|integrations/(bitsandbytes|gptq|awq|hqq|quanto|torchao|fp_quant|compressed|fbgemm|mxfp4|eetq|higgs|spqr|vptq|bitnet|finegrained))', 'quantization'),
 (r'^src/transformers/integrations/(gguf|ggml)|gguf', 'gguf'), (r'^src/transformers/(integrations/(flash|sdpa|flex|hub_kernels|npu)|kernels/|modeling_flash)', 'attention / kernels'),
 (r'^src/transformers/pipelines/', 'pipelines'), (r'^src/transformers/(cli|commands)/', 'serving / continuous batching'),
 (r'^src/transformers/(image_processing|video_processing|feature_extraction|processing_utils|audio_utils|image_utils|video_utils)', 'processors (image/video/audio)'),
 (r'^src/transformers/(cache_utils|masking_utils)', 'cache / masks'), (r'^src/transformers/(modeling_utils|core_model_loading|conversion_mapping|configuration_utils|safetensors|modeling_rope)', 'core / loading'),
 (r'^src/transformers/(distributed|integrations/(tensor_parallel|fsdp|deepspeed|accelerate|moe))', 'distributed'),
 (r'^src/transformers/exporters|executorch|onnx', 'exporters'), (r'^src/transformers/layer_execution', 'layer execution'), (r'^src/transformers/loss/', 'trainer'), (r'^src/transformers/(testing_utils)', 'tests'), (r'^src/transformers/(utils/|__init__|hf_argparser|activations|pytorch_utils|modeling_outputs|dependency_versions)', 'core / utils'), (r'^src/transformers/image_transforms', 'processors (image/video/audio)'), (r'^src/transformers/integrations/', 'integrations (other)'), (r'^docs/', 'docs'), (r'^tests/', 'tests'),
 (r'^(\.github|\.circleci|docker|utils|benchmark|Makefile|setup|pyproject)', 'CI / tooling'), (r'^examples/', 'examples'), (r'.', 'other')]
ISSUE_KW = [(r'(?i)\bgguf\b', 'gguf'), (r'(?i)export|onnx|executorch|torch\.compile|compile', 'exporters'), (r'(?i)For(Sequence|Token)Classification|QuestionAnswering|classification head|detection head|\bheads?\b', 'models'), (r'(?i)\bloss\b|label smoothing|muon|save_strategy|optimi[sz]er', 'trainer'), (r'(?i)typing|FloatTensor|RopeParameters|import_utils', 'core / utils'), (r'(?i)quantiz|bitsandbytes|bnb|awq|gptq|fp8|int4|int8|torchao|mxfp4|compressed.tensors', 'quantization'),
 (r'(?i)tokeni[sz]|chat.template|special token|vocab', 'tokenization'), (r'(?i)trainer|training|fine.?tun|deepspeed|fsdp|gradient|optimizer|lr sched', 'trainer'),
 (r'(?i)generat|beam|sampling|logits|stopping|streamer|speculat|assist', 'generation'), (r'(?i)\bcache\b|kv.cache|past_key|mask', 'cache / masks'),
 (r'(?i)flash.?attn|flash.?attention|sdpa|flex.?attention|kernel|attention impl', 'attention / kernels'),
 (r'(?i)tensor.parallel|\btp\b|expert.parallel|distributed|multi.?gpu|device_map|accelerate', 'distributed'),
 (r'(?i)pipeline', 'pipelines'), (r'(?i)serve|continuous.batch|generate_batch|\bcli\b', 'serving / continuous batching'),
 (r'(?i)image.?processor|processor|feature.extract|video|audio|vision', 'processors (image/video/audio)'),
 (r'(?i)\bdocs?\b|documentation|translat|typo', 'docs'), (r'(?i)from_pretrained|save_pretrained|load|safetensors|checkpoint|hub', 'core / loading'),
 (r'(?i)\btests?\b|\bci\b|flaky', 'tests')]
def pr_area(p):
    files = [f['path'] for f in (p.get('files') or {}).get('nodes', [])]
    c = Counter()
    for f in files:
        for rx, a in AREA_RULES:
            if re.search(rx, f): c[a] += 1; break
    if not c: return 'other', files
    top = c.most_common(1)[0][0]
    if top == 'tests' and len(c) > 1: top = [a for a, _ in c.most_common() if a != 'tests'][0]
    return top, files
def models_in(files, title):
    s = {m.group(1) for f in files for m in [re.match(r'src/transformers/models/([^/]+)/', f)] if m}
    sq = re.sub(r'[^a-z0-9]', '', title.lower())
    s |= {m for m in MODELS if len(m.replace('_','')) > 3 and re.search(r'(?<![a-z0-9])' + re.escape(m.replace('_','')) + r'(?![a-z0-9])', re.sub(r'[-_ ]', '', title.lower()))}
    s |= {m for m in MODELS if len(m) > 4 and '_' in m and m.replace('_','') in sq}
    return s - {'auto', 'deprecated'}
rows = []
for p in D['prs']:
    area, files = pr_area(p); t = p['title']
    newdirs = {m.group(1) for f in files for m in [re.match(r'src/transformers/models/([^/]+)/modeling_', f)] if m and m.group(1) not in MODELS}
    kind = 'new model' if newdirs or re.search(r'(?i)^\[?add(ing)? (support for )?[A-Z]|new model', t) and area == 'models' else \
           'fix' if re.search(r'(?i)\bfix|bug|error|crash|wrong|broken|regress', t) else \
           'docs' if area == 'docs' else 'CI / tooling' if area in ('CI / tooling', 'tests') else 'feature / refactor'
    rd = p.get('reviewDecision')
    status = 'draft' if p['isDraft'] else 'approved' if rd == 'APPROVED' else 'changes requested' if rd == 'CHANGES_REQUESTED' else \
             'conflicting' if p.get('mergeable') == 'CONFLICTING' else 'awaiting review'
    au = (p['author'] or {}).get('login') or ''
    who = 'bot' if BOTS.search(au) else 'HF' if p['authorAssociation'] in HF else 'first-timer' if p['authorAssociation'] in ('FIRST_TIME_CONTRIBUTOR', 'FIRST_TIMER', 'NONE') else 'contributor'
    rows.append(dict(k='PR', n=p['number'], t=t, area=area, kind=kind, status=status, who=who, au=au, age=age(p['createdAt']), idle=age(p['updatedAt']),
                     size=p['additions'] + p['deletions'], models=sorted(models_in(files, t)), lab=[l['name'] for l in p['labels']['nodes']], react=p['reactions']['totalCount']))
for i in D['issues']:
    t = i['title']; lab = [l['name'] for l in i['labels']['nodes']]
    area = 'models' if 'New model' in lab else next((a for rx, a in ISSUE_KW if re.search(rx, t + ' ' + ' '.join(lab))), 'models' if models_in([], t) else 'other')
    kind = 'new model' if 'New model' in lab else 'feature' if 'Feature request' in lab else 'bug' if ('bug' in lab or re.search(r'(?i)error|bug|fail|crash|wrong|broken|not work|issue with|incorrect|mismatch|regression|nan\b|oom', t)) else \
           'docs' if area == 'docs' else 'tracker / RFC' if re.search(r'(?i)rfc|tracker|roadmap|\[.*(call|community).*\]|meta', t) else 'question / other'
    au = (i['author'] or {}).get('login') or ''
    who = 'bot' if BOTS.search(au) else 'HF' if i['authorAssociation'] in HF else 'community'
    status = 'active <14d' if age(i['updatedAt']) < 14 else 'idle 14–90d' if age(i['updatedAt']) < 90 else 'idle >90d'
    rows.append(dict(k='issue', n=i['number'], t=t, area=area, kind=kind, status=status, who=who, au=au, age=age(i['createdAt']), idle=age(i['updatedAt']),
                     size=i['comments']['totalCount'], models=sorted(models_in([], t)), lab=lab, react=i['reactions']['totalCount']))
json.dump(rows, open('land_rows.json', 'w'))
P = [r for r in rows if r['k'] == 'PR']; I = [r for r in rows if r['k'] == 'issue']
bucket = lambda d: '<30d' if d < 30 else '30–90d' if d < 90 else '90–180d' if d < 180 else '180d–1y' if d < 365 else '>1y'
AGE = ['<30d', '30–90d', '90–180d', '180d–1y', '>1y']
def bars(title, cnt, order=None, total=None, key=None):
    items = [(k, cnt.get(k, 0)) for k in order] if order else cnt.most_common()
    mx = max([v for _, v in items] or [1]); total = total or sum(cnt.values())
    li = ''.join(f"<tr data-f='{key}' data-v='{H.escape(k)}' title='{H.escape(k)}: {v} ({100*v/total:.0f}%)'><td class='l'>{H.escape(k)}</td>"
                 f"<td class='b'><span style='width:{100*v/mx:.1f}%'></span></td><td class='v'>{v}</td><td class='p'>{100*v/total:.0f}%</td></tr>" for k, v in items if v)
    return f"<section class='card'><h3>{title}</h3><table class='bars'>{li}</table></section>"
mc = Counter(m for r in rows for m in r['models']); mcp = Counter(m for r in P for m in r['models']); mci = Counter(m for r in I for m in r['models'])
topm = ''.join(f"<tr data-f='model' data-v='{m}'><td class='l'>{m}</td><td class='b'><span style='width:{100*v/mc.most_common(1)[0][1]:.0f}%'></span></td><td class='v'>{v}</td><td class='p'>{mcp[m]} PR · {mci[m]} iss</td></tr>" for m, v in mc.most_common(20))
tiles = (f"<div class='tiles'><div class='tile'><b>{len(I)}</b>open issues</div><div class='tile'><b>{len(P)}</b>open PRs</div>"
         f"<div class='tile'><b>{sum(r['status']=='awaiting review' for r in P)}</b>PRs awaiting review</div>"
         f"<div class='tile'><b>{sum(r['status']=='approved' for r in P)}</b>PRs approved, not merged</div>"
         f"<div class='tile'><b>{sum(r['kind']=='bug' for r in I)}</b>bug reports</div>"
         f"<div class='tile'><b>{sum(r['idle']<14 for r in rows)}</b>active in last 14d</div></div>")
SEC = (tiles +
 "<h2>Pull requests</h2><div class='grid'>" + bars('By area (from files touched)', Counter(r['area'] for r in P), key='area') + bars('By status', Counter(r['status'] for r in P), ['awaiting review','approved','changes requested','conflicting','draft'], key='status') +
 bars('By kind', Counter(r['kind'] for r in P), key='kind') + bars('Author', Counter(r['who'] for r in P), ['HF','contributor','first-timer','bot'], key='who') + bars('Age (opened)', Counter(bucket(r['age']) for r in P), AGE) + "</div>" +
 "<h2>Issues</h2><div class='grid'>" + bars('By area (labels + title)', Counter(r['area'] for r in I), key='area') + bars('By kind', Counter(r['kind'] for r in I), key='kind') +
 bars('Activity', Counter(r['status'] for r in I), ['active <14d','idle 14–90d','idle >90d'], key='status') + bars('Author', Counter(r['who'] for r in I), ['HF','community','bot'], key='who') + bars('Age (opened)', Counter(bucket(r['age']) for r in I), AGE) + "</div>" +
 "<h2>Most-touched models (PRs + issues)</h2><div class='grid'><section class='card wide'><table class='bars'>" + topm + "</table></section></div>")
AREAS = sorted({r['area'] for r in rows})
page = """<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Open Backlog Map</title>
<style>
:root{--bg:#fafaf9;--fg:#1c1917;--mut:#78716c;--card:#fff;--bd:#e7e5e4;--bar:#2563eb;--acc:#2563eb}
@media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1c1917;--fg:#f5f5f4;--mut:#a8a29e;--card:#292524;--bd:#44403c;--bar:#60a5fa;--acc:#60a5fa}}
:root[data-theme="dark"]{--bg:#1c1917;--fg:#f5f5f4;--mut:#a8a29e;--card:#292524;--bd:#44403c;--bar:#60a5fa;--acc:#60a5fa}
body{background:var(--bg);color:var(--fg);font:14px/1.45 system-ui,sans-serif;margin:0 auto;padding:16px;max-width:1200px}
h1{font-size:20px;margin:4px 0}h2{font-size:16px;margin:24px 0 8px}h3{font-size:13px;margin:0 0 6px;color:var(--mut);font-weight:600}
.tiles{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:8px;margin:12px 0}.tile{background:var(--card);border:1px solid var(--bd);border-radius:8px;padding:10px;color:var(--mut);font-size:12px}.tile b{display:block;font-size:26px;color:var(--fg)}
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(340px,1fr));gap:10px}.card{background:var(--card);border:1px solid var(--bd);border-radius:8px;padding:10px 12px}.wide{grid-column:1/-1}
.bars{width:100%;border-collapse:collapse}.bars tr{cursor:pointer}.bars tr:hover td{background:color-mix(in srgb,var(--acc) 8%,transparent)}.bars td{padding:2px 4px;font-size:12px;white-space:nowrap}
.l{width:34%;overflow:hidden;text-overflow:ellipsis;max-width:180px}.b{width:46%}.b span{display:block;height:10px;background:var(--bar);border-radius:0 4px 4px 0;min-width:2px}.v{text-align:right;font-variant-numeric:tabular-nums}.p{text-align:right;color:var(--mut)}
#flt{margin:16px 0 6px;display:flex;gap:8px;flex-wrap:wrap;align-items:center}select,input{background:var(--card);color:var(--fg);border:1px solid var(--bd);border-radius:4px;padding:4px}
#tbl{width:100%;border-collapse:collapse;font-size:12px}#tbl th,#tbl td{border-bottom:1px solid var(--bd);padding:3px 6px;text-align:left;vertical-align:top}#tbl th{cursor:pointer;position:sticky;top:0;background:var(--bg)}
a{color:var(--acc)}.wrap{overflow-x:auto}
</style></head><body>
<h1>transformers open backlog — what's left</h1><div style="color:var(--mut)">as of 2026-10-21 · after the Oct 20–21 cleanup · click any bar to filter the table below</div>
__SEC__
<h2>Every open item</h2><div id="flt">
<select id="fk"><option value="">PRs + issues</option><option>PR</option><option>issue</option></select>
<select id="fa"><option value="">all areas</option>__AREAS__</select>
<input id="fq" placeholder="search title / model / author"><span id="fc" style="color:var(--mut)"></span><button id="fx">clear</button></div>
<div class="wrap"><table id="tbl"><thead><tr><th data-s="k">type</th><th data-s="n">#</th><th data-s="t">title</th><th data-s="area">area</th><th data-s="kind">kind</th><th data-s="status">status</th><th data-s="who">author</th><th data-s="age">age d</th><th data-s="idle">idle d</th><th data-s="models">models</th></tr></thead><tbody></tbody></table></div>
<script>
const ROWS=__ROWS__;let F={},sortK='idle',dir=1;
const U='https://github.com/huggingface/transformers/';
function draw(){const k=fk.value,a=fa.value,q=fq.value.toLowerCase();
 let r=ROWS.filter(x=>(!k||x.k===k)&&(!a||x.area===a)&&Object.entries(F).every(([f,v])=>f==='model'?x.models.includes(v):x[f]===v)&&(!q||(x.t+' '+x.models.join(' ')+' '+x.au).toLowerCase().includes(q)));
 r.sort((x,y)=>{const A=x[sortK],B=y[sortK];return (A>B?1:A<B?-1:0)*dir});
 fc.textContent=r.length+' items'+(Object.keys(F).length?' · filter: '+Object.entries(F).map(e=>e.join('=')).join(', '):'');
 const tb=document.querySelector('#tbl tbody');tb.innerHTML='';
 for(const x of r.slice(0,1500)){const tr=document.createElement('tr');
  const cells=[x.k,'',x.t,x.area,x.kind,x.status,x.who+(x.au?' @'+x.au:''),x.age,x.idle,x.models.join(', ')];
  cells.forEach((c,i)=>{const td=document.createElement('td');if(i===1){const l=document.createElement('a');l.href=U+(x.k==='PR'?'pull/':'issues/')+x.n;l.target='_blank';l.textContent='#'+x.n;td.appendChild(l)}else td.textContent=c;tr.appendChild(td)});tb.appendChild(tr)}}
document.querySelectorAll('.bars tr').forEach(tr=>tr.onclick=()=>{const f=tr.dataset.f,v=tr.dataset.v;if(!f||f==='undefined')return;
 const sec=tr.closest('.grid')?.previousElementSibling?.textContent||'';if(sec.startsWith('Pull'))fk.value='PR';else if(sec.startsWith('Issues'))fk.value='issue';
 if(f==='area'){fa.value=v;delete F.area}else F[f]=v;draw();document.getElementById('flt').scrollIntoView({behavior:'smooth'})});
document.querySelectorAll('#tbl th').forEach(th=>th.onclick=()=>{const s=th.dataset.s;dir=sortK===s?-dir:1;sortK=s;draw()});
fk.onchange=fa.onchange=fq.oninput=draw;fx.onclick=()=>{F={};fk.value='';fa.value='';fq.value='';draw()};draw();
</script></body></html>"""
page = page.replace('__SEC__', SEC).replace('__AREAS__', ''.join(f'<option>{H.escape(a)}</option>' for a in AREAS)).replace('__ROWS__', json.dumps(rows).replace('</', '<\\/'))
open('backlog.html', 'w').write(page)
print(len(I), 'issues', len(P), 'PRs')
print('PR area', Counter(r['area'] for r in P).most_common()); print('PR status', Counter(r['status'] for r in P)); print('PR kind', Counter(r['kind'] for r in P))
print('issue area', Counter(r['area'] for r in I).most_common()); print('issue kind', Counter(r['kind'] for r in I))
print('top models', mc.most_common(12))