Spaces:
Running
Running
File size: 4,112 Bytes
6749a8b a564d6f 6749a8b a564d6f 6749a8b a564d6f 65f90ba 6749a8b a564d6f 6749a8b a564d6f 65f90ba a564d6f 6749a8b a564d6f 8e2de47 a564d6f 6749a8b a564d6f 6749a8b a564d6f 8e2de47 a564d6f 6749a8b a564d6f 6749a8b a564d6f 61dad6b a564d6f 6749a8b 65f90ba 6749a8b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | #!/usr/bin/env python3
"""Move submissions from questions/pending/*.jsonl into questions/corpus.jsonl,
replacing each cb-pending placeholder with the next free cb-XXXX id. A
maintainer runs this on main after merging one or more submission PRs, never
on a PR branch (two branches would both claim the same next id):
python scripts/assign_ids.py
Pending files are processed in submission order: the timestamp of the git
commit that first added each file, then file name as a tie-breaker. Appends
to corpus.jsonl, deletes the processed pending files. Run
validate_questions.py afterward.
"""
import json
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
CORPUS = ROOT / "questions" / "corpus.jsonl"
PENDING_DIR = ROOT / "questions" / "pending"
ID_PATTERN = re.compile(r"^cb-(\d{4,})$")
def first_commit_time(path: Path) -> int:
"""Unix timestamp of the commit that first added `path`, or a large
sentinel if the file is not in git history yet (uncommitted), so
committed submissions always come before uncommitted ones."""
try:
out = subprocess.run(
["git", "log", "--diff-filter=A", "--follow", "--format=%at", "--", str(path)],
cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True,
).stdout.split()
return int(out[-1]) if out else 2**62
except (subprocess.CalledProcessError, FileNotFoundError, ValueError):
return 2**62
def main() -> int:
# Load existing corpus (if any)
items = []
if CORPUS.exists():
# split("\n"), not splitlines(): see validate_questions.py.
lines = [line.strip() for line in CORPUS.read_text(encoding="utf-8").split("\n") if line.strip()]
items = [json.loads(line) for line in lines]
used = set()
for item in items:
match = ID_PATTERN.match(item["id"])
if not match:
print(f"corpus.jsonl: unexpected id {item['id']!r}. Fix it before assigning.")
return 1
used.add(int(match.group(1)))
# Discover pending files in submission order (git add time, then name)
pending_files = []
if PENDING_DIR.exists():
pending_files = sorted(
PENDING_DIR.glob("*.jsonl"),
key=lambda p: (first_commit_time(p), p.name),
)
if not pending_files:
print("No pending files found in questions/pending/. Nothing to do.")
return 0
next_id = max(used, default=0) + 1
assigned = 0
new_items = []
for p in pending_files:
for lineno, line in enumerate(p.read_text(encoding="utf-8").split("\n"), start=1):
line = line.strip()
if not line:
continue
item = json.loads(line)
if item.get("id") != "cb-pending":
print(f"pending/{p.name}:{lineno}: id must be 'cb-pending', got {item.get('id')!r}. Nothing written.")
return 1
while next_id in used:
next_id += 1
item["id"] = f"cb-{next_id:04d}"
used.add(next_id)
assigned += 1
new_items.append(item)
print(f"Assigned {item['id']} from pending/{p.name}: {item['question'][:60]!r}")
if assigned == 0:
print("No cb-pending entries found in pending files. Nothing to do.")
return 0
# Append to (or create) corpus.jsonl
all_items = items + new_items
# newline="\n": text mode would write CRLF on Windows, and the corpus is committed with LF.
CORPUS.write_text("\n".join(json.dumps(item) for item in all_items) + "\n", encoding="utf-8", newline="\n")
# Remove processed pending files
for p in pending_files:
p.unlink()
print(f"\n{assigned} id(s) assigned and appended to {CORPUS.name} from {len(pending_files)} pending file(s). Now run scripts/validate_questions.py.")
return 0
if __name__ == "__main__":
# Questions can be in any script; a console that cannot show it must not crash the run.
sys.stdout.reconfigure(errors="backslashreplace")
raise SystemExit(main())
|