| |
| """Compare two Clef servers answer for answer, for example BF16 and FP8. |
| |
| python3 parity.py run --label bf16 --out bf16.json |
| python3 parity.py run --label fp8 --out fp8.json |
| python3 parity.py compare bf16.json fp8.json --report parity-report.md |
| |
| `run` sends a fixed set of requests, one at a time, and saves every answer: |
| 54 text requests (18 states x 3 question sets: 6 questions per state, in 5 |
| languages, with one long state) and 5 Commons images. Add your own requests with |
| --requests, one JSON request body per line. |
| |
| `compare` writes a short Markdown report: how far each probability moved, and |
| how often the answer changed. It says PASS or FAIL against two limits. |
| |
| `report` does the same for a run_all.sh results directory, and adds the |
| quantization details and the GPU memory that vLLM logged. |
| |
| Standard library only (Python >= 3.9). |
| """ |
|
|
| import argparse |
| import base64 |
| import datetime |
| import json |
| import os |
| import re |
| import socket |
| import statistics |
| import sys |
| import urllib.request |
|
|
| URL = "http://localhost:8080/v1/models/clef-flash:predict" |
| USER_AGENT = "clef-parity/0.1 (WMF ML team)" |
| LOCAL = urllib.request.build_opener(urllib.request.ProxyHandler({})) |
|
|
| EDITS = [ |
| ("edit_promo", "Edit replaced the article lead with promotional language and removed three citations."), |
| ("edit_typo", "Edit fixed two typos and a broken link in the History section."), |
| ("edit_meme", "Edit replaced the infobox image with an unrelated meme and added 'LOL' to the first sentence."), |
| ("edit_blank", "Edit removed the entire Criticism section without an edit summary."), |
| ("edit_cite", "Edit added a citation to a peer-reviewed journal for the population figure."), |
| ("edit_casino", "Edit added a link to an online casino in the External links section."), |
| ] |
| STATES = EDITS + [ |
| ("json_anon", {"user": "anonymous", "account_age_days": 0, "bytes_changed": -4210, "summary": "", |
| "page": "Climate change", "tags": ["mobile edit", "blanking"]}), |
| ("json_veteran", {"user": "registered", "account_age_days": 2900, "edit_count": 15400, "bytes_changed": 37, |
| "summary": "copyedit", "page": "Mount Olympus"}), |
| ("article_restaurant", "Article about a local restaurant, written by its owner, with no independent sources."), |
| ("article_poet", "Stub article about a 19th-century Greek poet with one reference to a national library catalogue."), |
| ("article_band", "Article about a garage band with 40 followers, sourced only to the band's own website."), |
| ("talk_hostile", "Talk page message: 'Stop reverting my edits, you clearly know nothing about this topic.'"), |
| ("talk_polite", "Talk page message: 'Thanks for the review. I added two sources. Could you take another look?'"), |
| ("el", "Η επεξεργασία αντικατέστησε την εισαγωγή του άρθρου με διαφημιστικό κείμενο."), |
| ("es", "La edición añadió una fuente académica y corrigió la fecha de nacimiento."), |
| ("de", "Die Bearbeitung entfernte alle Einzelnachweise aus dem Artikel."), |
| ("ja", "この編集は、出典のない誹謗中傷を人物記事に追加しました。"), |
| ("long_history", "Edit history of one article over a month:\n" |
| + "\n".join(f"{i + 1}. {EDITS[i % len(EDITS)][1]}" for i in range(60))), |
| ] |
|
|
| QUESTION_SETS = [ |
| ("revert", {"revert": {"type": "noul", "instructions": "Should this edit be reverted?"}}), |
| ("action_severity", { |
| "action": {"type": "choice", "instructions": "What should happen to this?", "criteria": { |
| "keep": "Notable enough and fixable by editing.", |
| "improve": "Needs sources and a neutral rewrite before it can stay.", |
| "delete": "Not notable and unlikely to become so."}}, |
| "severity": {"type": "score", "instructions": "How severe is the problem?", |
| "criteria": ["Minor", "Moderate", "Severe"]}, |
| }), |
| ("tone_sources_quality", { |
| "tone": {"type": "choice", "instructions": "What is the tone?", "criteria": { |
| "neutral": "Neutral and encyclopedic.", "promotional": "Advertises or praises the subject.", |
| "hostile": "Attacks a person or group.", "other": "None of the above."}}, |
| "needs_sources": {"type": "noul", "instructions": "Does this need more or better sources?"}, |
| "quality": {"type": "score", "instructions": "How good is this contribution?", |
| "criteria": ["Very poor", "Poor", "Acceptable", "Good", "Excellent"]}, |
| }), |
| ] |
|
|
| |
| IMAGES = [ |
| ("portrait", "https://upload.wikimedia.org/wikipedia/commons/thumb/5/57/Jimmy_Wales_by_Philip_Romano_%283x4_cropped%29.jpg/960px-Jimmy_Wales_by_Philip_Romano_%283x4_cropped%29.jpg"), |
| ("building", "https://thumb.wikimedia.org/wikipedia/commons/thumb/a/ad/Parthenon_from_west.jpg/960px-Parthenon_from_west.jpg"), |
| ("animal", "https://thumb.wikimedia.org/wikipedia/commons/thumb/4/4d/Cat_November_2010-1a.jpg/960px-Cat_November_2010-1a.jpg"), |
| ("painting", "https://thumb.wikimedia.org/wikipedia/commons/thumb/f/f9/Declaration_of_Independence_%281819%29%2C_by_John_Trumbull.jpg/960px-Declaration_of_Independence_%281819%29%2C_by_John_Trumbull.jpg"), |
| ("logo", "https://thumb.wikimedia.org/wikipedia/commons/thumb/8/80/Wikipedia-logo-v2.svg/960px-Wikipedia-logo-v2.svg.png"), |
| ] |
| IMAGE_QUESTIONS = { |
| "infobox_fit": {"type": "choice", "instructions": "Is this suitable as an infobox image?", "criteria": { |
| "yes": "Good framing and clear subject.", "maybe": "Usable with reservations.", "no": "Not suitable."}}, |
| "face_visible": {"type": "noul", "instructions": "Is a human face clearly visible?"}, |
| "image_quality": {"type": "score", "instructions": "How good is the image quality?", |
| "criteria": ["Poor", "Acceptable", "Good"]}, |
| } |
|
|
|
|
| def build_requests(args): |
| requests = [(f"text/{sid}/{qid}", {"state": state, "questions": questions}) |
| for sid, state in STATES for qid, questions in QUESTION_SETS] |
| if not args.no_images: |
| os.makedirs(args.image_dir, exist_ok=True) |
| for name, url in IMAGES: |
| path = os.path.join(args.image_dir, name + os.path.splitext(url)[1]) |
| if not os.path.exists(path): |
| request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) |
| with urllib.request.urlopen(request, timeout=60) as response, open(path, "wb") as f: |
| f.write(response.read()) |
| with open(path, "rb") as f: |
| image = base64.b64encode(f.read()).decode() |
| requests.append((f"image/{name}", {"state": "Assess this image for use in a Wikipedia article.", |
| "images": [image], "questions": IMAGE_QUESTIONS})) |
| if args.requests: |
| with open(args.requests) as f: |
| for number, line in enumerate(f, 1): |
| if line.strip(): |
| body = json.loads(line) |
| requests.append((f"file/{body.pop('id', number)}", body)) |
| return requests |
|
|
|
|
| def run(args): |
| requests = build_requests(args) |
| results = {} |
| for number, (rid, body) in enumerate(requests, 1): |
| try: |
| request = urllib.request.Request(args.url, json.dumps(body).encode(), {"Content-Type": "application/json"}) |
| with LOCAL.open(request, timeout=args.timeout) as response: |
| data = json.load(response) |
| results[rid] = {"answers": data["answers"], "input_tokens": data["usage"]["input_tokens"]} |
| except Exception as e: |
| results[rid] = {"error": repr(e)[:300]} |
| print(f"[{number}/{len(requests)}] {'ERROR' if 'error' in results[rid] else 'ok '} {rid}", flush=True) |
| with open(args.out, "w") as f: |
| json.dump({"label": args.label, "url": args.url, "results": results}, f, ensure_ascii=False, indent=1) |
| print(f"Saved {len(results)} results to {args.out}") |
|
|
|
|
| def probabilities(answer): |
| """Return the probability of each option of one answer.""" |
| if answer["type"] == "noul": |
| return {"yes": answer["noul"]} |
| return answer["probabilities"] |
|
|
|
|
| def decision(answer): |
| """Return the answer as a person reads it, and its probability. |
| |
| That is yes/no at 0.5, the choice, or the most likely score level. |
| """ |
| if answer["type"] == "noul": |
| p = answer["noul"] |
| return ("yes", p) if p >= 0.5 else ("no", 1 - p) |
| if answer["type"] == "choice": |
| return answer["choice"], answer["probabilities"][answer["choice"]] |
| level = max(answer["probabilities"], key=answer["probabilities"].get) |
| return level, answer["probabilities"][level] |
|
|
|
|
| def parity_lines(path_a, path_b, max_change_limit, min_agreement, sub="##", verdict=True): |
| """Compare two saved runs. Return the Markdown lines and whether they pass. |
| |
| With verdict=False, the lines give the numbers without PASS/FAIL or the limits. |
| """ |
| with open(path_a) as f: |
| a = json.load(f) |
| with open(path_b) as f: |
| b = json.load(f) |
| la, lb = a["label"], b["label"] |
| changes, flips, errors, token_mismatch = [], [], [], [] |
| same = {"text": [0, 0], "image": [0, 0], "file": [0, 0]} |
| for rid in sorted(set(a["results"]) & set(b["results"])): |
| ra, rb = a["results"][rid], b["results"][rid] |
| if "error" in ra or "error" in rb: |
| errors.append(rid) |
| continue |
| if ra["input_tokens"] != rb["input_tokens"]: |
| token_mismatch.append(rid) |
| kind = rid.split("/")[0] |
| for qid, qa in ra["answers"].items(): |
| qb = rb["answers"][qid] |
| pa, pb = probabilities(qa), probabilities(qb) |
| for option in pa: |
| changes.append((abs(pa[option] - pb[option]), kind, rid, qid, option, pa[option], pb[option])) |
| same[kind][1] += 1 |
| (da, pda), (db, pdb) = decision(qa), decision(qb) |
| if da == db: |
| same[kind][0] += 1 |
| else: |
| flips.append((rid, qid, da, pda, db, pdb)) |
|
|
| def stats(kind=None): |
| values = sorted(c[0] for c in changes if kind in (None, c[1])) |
| if not values: |
| return None |
| p99 = values[max(0, int(0.99 * len(values)) - 1)] |
| return statistics.mean(values), p99, values[-1] |
|
|
| total_same = sum(s for s, _ in same.values()) |
| total = sum(t for _, t in same.values()) |
| max_change = stats()[2] if changes else float("nan") |
| agreement = total_same / total if total else 0.0 |
| passed = not errors and not token_mismatch and max_change <= max_change_limit and agreement >= min_agreement |
|
|
| if verdict: |
| summary = ( |
| f"**{'PASS' if passed else 'FAIL'}**: largest change {max_change:.4f} (limit {max_change_limit}), " |
| f"same answer {agreement:.1%} (limit {min_agreement:.0%}), " |
| ) |
| else: |
| summary = f"Largest change {max_change:.4f}, same answer {agreement:.1%}, " |
| lines = [ |
| summary + f"{len(errors)} failed requests, {len(token_mismatch)} with different token counts.", |
| "", |
| "| Requests | Questions | Mean change | 99th percentile | Largest change | Same answer |", |
| "|---|---|---|---|---|---|", |
| ] |
| for kind in ("text", "image", "file"): |
| if same[kind][1]: |
| mean, p99, top = stats(kind) |
| lines.append(f"| {kind} | {same[kind][1]} | {mean:.4f} | {p99:.4f} | {top:.4f} | " |
| f"{same[kind][0]}/{same[kind][1]} |") |
| lines += [ |
| "", |
| f"*Change* is the difference between the {la} and {lb} probability of one option (0 to 1). " |
| "*Same answer* means the same choice, the same yes/no at 0.5, or the same most likely score level.", |
| "", |
| f"{sub} Largest changes", |
| "", |
| f"| Request | Question | Option | {la} | {lb} | Change |", |
| "|---|---|---|---|---|---|", |
| ] |
| for change, _, rid, qid, option, pa, pb in sorted(changes, reverse=True)[:10]: |
| lines.append(f"| {rid} | {qid} | {option} | {pa:.4f} | {pb:.4f} | {change:.4f} |") |
| if flips: |
| lines += ["", f"{sub} Answers that changed", "", f"| Request | Question | {la} | {lb} |", "|---|---|---|---|"] |
| for rid, qid, da, pa, db, pb in flips: |
| lines.append(f"| {rid} | {qid} | {da} ({pa:.2f}) | {db} ({pb:.2f}) |") |
| if errors or token_mismatch: |
| lines += ["", f"{sub} Problems", ""] |
| lines += [f"- Failed request: {rid}" for rid in errors] |
| lines += [f"- Different token count: {rid}" for rid in token_mismatch] |
| return lines, passed |
|
|
|
|
| def write_report(path, lines, passed): |
| text = "\n".join(lines) + "\n" |
| with open(path, "w") as f: |
| f.write(text) |
| print(text) |
| print(f"Wrote {path}") |
| sys.exit(0 if passed else 1) |
|
|
|
|
| def compare(args): |
| lines, passed = parity_lines(args.a, args.b, args.max_change, args.min_agreement) |
| with open(args.b) as f: |
| lb = json.load(f)["label"] |
| with open(args.a) as f: |
| la = json.load(f)["label"] |
| write_report(args.report, [f"# Clef parity: {lb} against {la}", ""] + lines, passed) |
|
|
|
|
| def server_facts(path): |
| """Read what vLLM logged about quantization and GPU memory.""" |
| facts = {"quantization": "?", "weights": "?", "kv_cache": "?"} |
| if not os.path.exists(path): |
| return facts |
| with open(path, errors="replace") as f: |
| log = f.read() |
| for key, pattern in ( |
| ("quantization", r"quantization=([\w-]+)"), |
| ("weights", r"Model loading took ([\d.]+ GiB)"), |
| ("kv_cache", r"Available KV cache memory: ([\d.]+ GiB)"), |
| ): |
| match = re.search(pattern, log) |
| if match: |
| facts[key] = match.group(1) |
| if facts["quantization"] == "None": |
| facts["quantization"] = "none (BF16)" |
| return facts |
|
|
|
|
| def report(args): |
| """Write one report for the whole run: quantization, GPU memory and answers.""" |
| d = args.results |
| parity, passed = parity_lines(os.path.join(d, "bf16.json"), os.path.join(d, "fp8.json"), |
| args.max_change, args.min_agreement, sub="###", verdict=not args.no_verdict) |
| when = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC") |
| lines = [ |
| "# Clef-Flash: FP8 against BF16", |
| "", |
| (f"**{'PASS' if passed else 'FAIL'}**. " if not args.no_verdict else "") + f"Run on {socket.gethostname()} on {when}.", |
| *([""] + args.note if args.note else []), |
| "", |
| ] |
| manifest_path = os.path.join(d, "MANIFEST.json") |
| if os.path.exists(manifest_path): |
| with open(manifest_path) as f: |
| m = json.load(f) |
| v = m["versions"] |
| lines += [ |
| "## Quantization", |
| "", |
| f"- Source: `{m['source']}`. Every input file matched Cloudflare's published hash.", |
| f"- Tool: llm-compressor {v['llmcompressor']}, scheme `{m['recipe']['scheme']}` " |
| f"(compressed-tensors {v['compressed_tensors']}, transformers {v['transformers']}, torch {v['torch']}).", |
| f"- {m['fp8_weights']} weights in FP8, relative error mean {m['relative_error_mean']:.2%}, " |
| f"max {m['relative_error_max']:.2%}. All other tensors are bit-identical to Cloudflare's.", |
| "", |
| ] |
| bf16, fp8 = server_facts(os.path.join(d, "server-bf16.log")), server_facts(os.path.join(d, "server-fp8.log")) |
| lines += [ |
| "## GPU memory, as logged by vLLM", |
| "", |
| "| | BF16 | FP8 |", |
| "|---|---|---|", |
| f"| Quantization | {bf16['quantization']} | {fp8['quantization']} |", |
| f"| Weights | {bf16['weights']} | {fp8['weights']} |", |
| f"| Free for KV cache | {bf16['kv_cache']} | {fp8['kv_cache']} |", |
| "", |
| "Speed on the MI210 means nothing here: it has no FP8 hardware, so vLLM emulates FP8.", |
| "", |
| "## Answers", |
| "", |
| *parity, |
| ] |
| write_report(args.out or os.path.join(d, "report.md"), lines, passed) |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| sub = parser.add_subparsers(dest="command", required=True) |
| p = sub.add_parser("run", help="Send the requests to one server and save the answers.") |
| p.add_argument("--label", required=True, help="Name of this server, for example bf16 or fp8.") |
| p.add_argument("--out", required=True) |
| p.add_argument("--url", default=URL) |
| p.add_argument("--requests", help="Extra requests: a JSONL file, one request body per line.") |
| p.add_argument("--image-dir", default="parity-images") |
| p.add_argument("--no-images", action="store_true") |
| p.add_argument("--timeout", type=float, default=300) |
| p.set_defaults(func=run) |
| c = sub.add_parser("compare", help="Compare two saved runs and write the report.") |
| c.add_argument("a", help="Reference run, for example bf16.json.") |
| c.add_argument("b", help="Candidate run, for example fp8.json.") |
| c.add_argument("--report", default="parity-report.md") |
| c.add_argument("--max-change", type=float, default=0.03) |
| c.add_argument("--min-agreement", type=float, default=0.99) |
| c.set_defaults(func=compare) |
| r = sub.add_parser("report", help="Write one report for a run_all.sh results directory.") |
| r.add_argument("results", help="Directory with bf16.json, fp8.json, MANIFEST.json and the server logs.") |
| r.add_argument("--note", action="append", help="A line to add under the title. Repeat for more lines.") |
| r.add_argument("--no-verdict", action="store_true", help="Leave out PASS/FAIL and the limits, for example to publish.") |
| r.add_argument("--out", help="Where to write the report. Default: report.md in the results directory.") |
| r.add_argument("--max-change", type=float, default=0.03) |
| r.add_argument("--min-agreement", type=float, default=0.99) |
| r.set_defaults(func=report) |
| args = parser.parse_args() |
| args.func(args) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
