code/sweep.py

import json, subprocess, sys, time

OUT = sys.argv[1]
Q = """
query($cursor:String){repository(owner:"huggingface",name:"transformers"){
 %s(states:OPEN, first:40, after:$cursor){ totalCount pageInfo{hasNextPage endCursor}
  nodes{ number title createdAt updatedAt author{login} %s
   timelineItems(first:60, itemTypes:[CROSS_REFERENCED_EVENT, CONNECTED_EVENT, MARKED_AS_DUPLICATE_EVENT]){ nodes{
    __typename
    ... on CrossReferencedEvent{ willCloseTarget source{ __typename ... on PullRequest{number title state merged mergedAt baseRefName}
         ... on Issue{number title state}}}
    ... on ConnectedEvent{ subject{ __typename ... on PullRequest{number title state merged mergedAt}}}
    ... on MarkedAsDuplicateEvent{ canonical{ __typename ... on Issue{number state} ... on PullRequest{number state merged}}}
   }}
   comments(last:8){nodes{author{login} body createdAt}}
  }}}}
"""
PR_EXTRA = "isDraft baseRefName closingIssuesReferences(first:10){nodes{number state closedByPullRequestsReferences(first:5, includeClosedPrs:true){nodes{number merged state}}}}"

def run(kind, extra):
    items, cursor, page, t0 = [], None, 0, time.time()
    while True:
        args = ["gh", "api", "graphql", "-f", f"query={Q % (kind, extra)}"]
        if cursor: args += ["-f", f"cursor={cursor}"]
        for attempt in range(4):
            r = subprocess.run(args, capture_output=True, text=True)
            if r.returncode == 0: break
            time.sleep(5)
        d = json.loads(r.stdout)["data"]["repository"][kind]
        items += d["nodes"]; page += 1
        total = d["totalCount"]; el = time.time() - t0
        eta = el / len(items) * (total - len(items))
        print(f"[{kind} {len(items)}/{total}] page {page} | elapsed {el:.0f}s | eta ~{eta:.0f}s", flush=True)
        if not d["pageInfo"]["hasNextPage"]: return items
        cursor = d["pageInfo"]["endCursor"]

data = {"issues": run("issues", ""), "prs": run("pullRequests", PR_EXTRA)}
json.dump(data, open(OUT, "w"))
print("done", flush=True)