| |
| |
| """HF Jobs wiki adherence + refusal + length suite (same probes as Gradio gate).""" |
|
|
| import argparse |
| import json |
| import re |
| import time |
| import urllib.parse |
| import urllib.request |
| from pathlib import Path |
|
|
| import torch |
| from transformers import AutoTokenizer |
|
|
| from generate import generate, load_train_module, resolve_model_dir |
|
|
| SYSTEM = "You are a helpful assistant. Answer clearly and correctly." |
| ABSTAIN_RE = re.compile(r"don'?t know|not enough|no information|based on the provided|don'?t have data", re.I) |
| UA = {"User-Agent": "ScugnizzContextGroundEval/1.0"} |
|
|
| WIKI_ADH = [ |
| ("Malbork Castle", "What is the largest castle by land area mentioned?", ["malbork"], False), |
| ("Mount Everest", "Which countries does the mountain border according to the text?", ["china", "nepal"], False), |
| ("Pacific Ocean", "What is the largest ocean?", ["pacific"], False), |
| ("Python (programming language)", "Who created Python?", ["guido", "van rossum"], False), |
| ("Wieliczka Salt Mine", "Near which city is the salt mine?", ["krak", "cracow"], False), |
| ("Axolotl", "Where is the axolotl native?", ["mexico"], False), |
| ("Antikythera mechanism", "In what year was it discovered?", ["1901"], False), |
| ("Tardigrade", "What other name are they known by?", ["water bear"], False), |
| ("Alan Turing", "What is Alan Turing best known for according to the text?", ["turing", "computer", "enigma", "machine"], False), |
| ("Photosynthesis", "What gas is released by photosynthesis?", ["oxygen"], False), |
| ] |
| WIKI_REF = [ |
| ("Malbork Castle", "How many visitors did Malbork Castle have in 2019?", True), |
| ("Wieliczka Salt Mine", "What is the current adult ticket price in PLN?", True), |
| ("Axolotl", "What is the typical pet lifespan of an axolotl in years?", True), |
| ("Mount Everest", "What was the name of the first dog to summit Everest?", True), |
| ("Pacific Ocean", "What is the exact deepest point of the Pacific in meters?", True), |
| ("Antikythera mechanism", "Who was the captain of the ship that found it?", True), |
| ("Tardigrade", "Are tardigrades found only in Antarctica?", True), |
| ("Alan Turing", "What did Turing eat for breakfast on 7 June 1954?", True), |
| ("Photosynthesis", "Who discovered photosynthesis in the year 1779?", True), |
| ("Python (programming language)", "How many times was Python 3.12 downloaded in 2024?", True), |
| ] |
| LENGTH = [ |
| ("short", "Mira Cole painted Dawn. Jonas Pike cast Echo Chamber in bronze. Rae Quinn filmed Night Bus.", |
| "Who painted Dawn?", ["mira cole"]), |
| ("short", "Lab note: water freezes at 40 degrees Celsius under the special protocol.", |
| "At what Celsius temperature does water freeze according to the note?", ["40"]), |
| ("short", "The Official Kanban Guide was put together by Susanne and Andreas Bartel of Kanban University.", |
| "Who put together the Official Guide?", ["susanne", "andreas", "bartel"]), |
| ("medium", ( |
| "Poland is known for several landmarks. The largest castle in the world by land area is " |
| "Malbork Castle, a brick fortress of the Teutonic Knights in northern Poland. Near Kraków, " |
| "the Wieliczka Salt Mine has underground chambers carved from salt over centuries. Further " |
| "south, the Błędowska Desert — sometimes called the Polish Sahara — is a stretch of inland " |
| "sand dunes. The Vistula is the longest river in Poland." |
| ), "What is sometimes called the Polish Sahara?", ["błędowska", "bledowska"]), |
| ("medium", ( |
| "Asteroid 243 Ida was visited by the Galileo spacecraft. Ida is about 31 kilometres long " |
| "and has a small moon named Dactyl. Dactyl was the first moon of an asteroid to be discovered. " |
| "The flyby occurred in 1993 during Galileo's journey to Jupiter. Ida belongs to the Koronis family." |
| ), "What is the name of Ida's moon?", ["dactyl"]), |
| ("medium", ( |
| "Lake Merín sits on the Brazil–Uruguay border. It is a large shallow lagoon used for fishing " |
| "and irrigation. The lake drains toward the Atlantic via a chain of wetlands. Local guides " |
| "note migratory birds in southern summer. The shoreline towns trade in rice and cattle." |
| ), "On which border does Lake Merín sit?", ["brazil", "uruguay"]), |
| ("long", ( |
| "The Antikythera mechanism is an ancient Greek hand-powered orrery, described as the oldest " |
| "known example of an analogue computer used to predict astronomical positions and eclipses " |
| "decades in advance. It was recovered in 1901 from the Antikythera shipwreck off the Greek " |
| "island of Antikythera. Technological artifacts of similar complexity did not appear again " |
| "until the 14th century, with mechanical astronomical clocks in Europe. The device is " |
| "remarkable for the level of miniaturisation and complexity of its parts, which is comparable " |
| "to that of 18th-century clocks. It has at least 30 gear wheels, although modeling work " |
| "suggests it may have had more. All known fragments of the Antikythera mechanism are kept " |
| "at the National Archaeological Museum in Athens. Researchers reconstructed the gear " |
| "functions using surface imaging and X-ray tomography. The mechanism also tracked the " |
| "four-year cycle of athletic games similar to the Olympiad, the cycle of the ancient Olympic Games." |
| ), "In what year was the mechanism recovered from the shipwreck?", ["1901"]), |
| ("long", ( |
| "An axolotl is a paedomorphic salamander closely related to the tiger salamander. Axolotls " |
| "are unusual among amphibians in that they reach adulthood without undergoing metamorphosis. " |
| "Instead of taking to land, adults remain aquatic and gilled. The species was originally " |
| "found in several lakes underlying what is now Mexico City, such as Lake Xochimilco and Lake " |
| "Chalco. These lakes were largely drained by Spanish settlers after the conquest of the Aztec " |
| "Empire, leading to the destruction of much of the axolotl's natural habitat. Axolotls should " |
| "not be confused with waterdogs, the larval stage of the closely related tiger salamanders, " |
| "which are widespread in much of North America and also occasionally become adult while aquatic. " |
| "The axolotl is currently critically endangered in the wild due to urbanization and pollution." |
| ), "Where was the species originally found?", ["mexico", "xochimilco"]), |
| ("long", ( |
| "Photosynthesis is a system of biological processes by which photosynthetic organisms, such as " |
| "most plants, algae, and cyanobacteria, convert light energy, typically from sunlight, into " |
| "the chemical energy necessary to fuel their metabolism. Photosynthesis usually refers to " |
| "oxygenic photosynthesis, a process that stores the chemical energy in carbohydrate molecules " |
| "synthesized from carbon dioxide and water. Most organisms that utilize photosynthesis to " |
| "produce oxygen are photoautotrophs; they produce their own food. In plants, algae, and " |
| "cyanobacteria, photosynthesis releases oxygen as a waste product. Photosynthesis is vital " |
| "for climate processes, as it captures carbon dioxide from the air and produces oxygen. " |
| "The rate of photosynthesis is determined by the available CO2, intensity of light, and " |
| "temperature. Photosynthetic organisms also synthesize other compounds such as proteins and lipids." |
| ), "What gas does photosynthesis release as a waste product?", ["oxygen"]), |
| ] |
|
|
|
|
| def wiki_plain(title, max_chars=1500): |
| url = ( |
| "https://en.wikipedia.org/w/api.php?action=query&prop=extracts&exintro=1&explaintext=1" |
| f"&titles={urllib.parse.quote(title)}&format=json" |
| ) |
| last = None |
| for i in range(5): |
| try: |
| req = urllib.request.Request(url, headers=UA) |
| with urllib.request.urlopen(req, timeout=40) as r: |
| data = json.load(r) |
| pages = data["query"]["pages"] |
| text = next(iter(pages.values())).get("extract") or "" |
| text = re.sub(r"\s+", " ", text).strip() |
| return text[:max_chars] |
| except Exception as e: |
| last = e |
| time.sleep(2 + i * 2) |
| raise RuntimeError(f"wiki failed {title}: {last}") |
|
|
|
|
| def score_ans(ans, needles, want_abstain): |
| a = (ans or "").strip() |
| abstain = bool(ABSTAIN_RE.search(a)) |
| if want_abstain: |
| return abstain, "abstain" if abstain else "invent" |
| if abstain: |
| return False, "abstain_on_answerable" |
| low = a.lower() |
| ok = all(n.lower() in low for n in needles) if needles else False |
| if not ok and needles and len(needles) > 1: |
| ok = any(n.lower() in low for n in needles) |
| return ok, "partial" if ok else "miss" |
| return ok, "ok" if ok else "miss" |
|
|
|
|
| def load_model(args, work_dir): |
| mod = load_train_module(work_dir) |
| model_dir = resolve_model_dir(args) |
| run_args = json.loads((model_dir / "args.json").read_text(encoding="utf-8")) |
| dev = "cuda" if torch.cuda.is_available() else "cpu" |
| tok = AutoTokenizer.from_pretrained(args.tokenizer) |
| if tok.pad_token is None: |
| tok.pad_token = tok.eos_token |
| cfg = mod.preset_config( |
| run_args.get("model_size", "1.7b"), |
| len(tok), |
| int(run_args.get("block_size", 4096)), |
| 0.0, |
| run_args.get("pcs_a", 0.8309193524478643), |
| run_args.get("pcs_b", 0.0), |
| ) |
| cfg.gradient_checkpointing = False |
| model = mod.ScugnizzDecoder(cfg).to(dev) |
| weights_name = args.weights_file.strip() or "model_final.pt" |
| wpath = model_dir / weights_name |
| if not wpath.is_file(): |
| |
| wpath = Path(weights_name) |
| weights = torch.load(wpath, map_location=dev, weights_only=True) |
| state = weights["model"] if isinstance(weights, dict) and "model" in weights else weights |
| model.load_state_dict(state, strict=True) |
| print(f"loaded {wpath}", flush=True) |
| return model, tok, dev, model_dir |
|
|
|
|
| def chat_prompt(tok, user_msg): |
| msgs = [ |
| {"role": "system", "content": SYSTEM}, |
| {"role": "user", "content": user_msg}, |
| ] |
| try: |
| return tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True) |
| except Exception: |
| return f"System: {SYSTEM}\nUser: {user_msg}\nAssistant:" |
|
|
|
|
| def main(): |
| p = argparse.ArgumentParser() |
| g = p.add_mutually_exclusive_group(required=True) |
| g.add_argument("--model-dir") |
| g.add_argument("--hub-repo") |
| p.add_argument("--hub-path", required=True) |
| p.add_argument("--local-dir", default="/tmp/scugnizz-model") |
| p.add_argument("--weights-file", default="model_final.pt", |
| help="Weight filename under hub-path (model_final.pt / model_pulse_2400.pt / …)") |
| p.add_argument("--tokenizer", default="meta-llama/Llama-3.2-1B") |
| p.add_argument("--out-dir", default="/tmp/context-ground-eval") |
| p.add_argument("--label", default="") |
| p.add_argument("--temperature", type=float, default=0.0) |
| p.add_argument("--top-k", type=int, default=0) |
| p.add_argument("--max-new-tokens", type=int, default=64) |
| args = p.parse_args() |
|
|
| work_dir = Path(__file__).resolve().parent |
| out_dir = Path(args.out_dir) |
| out_dir.mkdir(parents=True, exist_ok=True) |
| label = args.label or f"{args.hub_path}/{args.weights_file}" |
|
|
| model, tok, dev, model_dir = load_model(args, work_dir) |
|
|
| def ask(msg): |
| prompt = chat_prompt(tok, msg) |
| return generate( |
| model, tok, prompt, args.max_new_tokens, args.temperature, args.top_k, dev |
| ).strip() |
|
|
| results = {"label": label, "hub_path": args.hub_path, "weights": args.weights_file, |
| "adh": [], "ref": [], "len": []} |
| cache = {} |
|
|
| print(f"=== wiki adherence ({label}) ===", flush=True) |
| for title, q, needles, _ in WIKI_ADH: |
| passage = cache.get(title) or wiki_plain(title) |
| cache[title] = passage |
| ans = ask(f"{passage}\n\n{q}") |
| ok, tag = score_ans(ans, needles, False) |
| results["adh"].append({"title": title, "q": q, "ok": ok, "tag": tag, "ans": ans[:240]}) |
| print(f"{'PASS' if ok else 'FAIL'} {title} [{tag}] → {ans[:120]!r}", flush=True) |
|
|
| print(f"=== wiki refusal ({label}) ===", flush=True) |
| for title, q, _ in WIKI_REF: |
| passage = cache.get(title) or wiki_plain(title) |
| cache[title] = passage |
| ans = ask(f"{passage}\n\n{q}") |
| ok, tag = score_ans(ans, None, True) |
| results["ref"].append({"title": title, "q": q, "ok": ok, "tag": tag, "ans": ans[:240]}) |
| print(f"{'PASS' if ok else 'FAIL'} {title} [{tag}] → {ans[:120]!r}", flush=True) |
|
|
| print(f"=== length suite ({label}) ===", flush=True) |
| for band, passage, q, needles in LENGTH: |
| ans = ask(f"{passage}\n\n{q}") |
| ok, tag = score_ans(ans, needles, False) |
| results["len"].append({"band": band, "q": q, "ok": ok, "tag": tag, "ans": ans[:240], "chars": len(passage)}) |
| print(f"{'PASS' if ok else 'FAIL'} {band}({len(passage)}) [{tag}] → {ans[:120]!r}", flush=True) |
|
|
| summary = { |
| "adh": sum(1 for r in results["adh"] if r["ok"]), |
| "ref": sum(1 for r in results["ref"] if r["ok"]), |
| "len": sum(1 for r in results["len"] if r["ok"]), |
| } |
| results["summary"] = summary |
| out_json = out_dir / "results.json" |
| out_json.write_text(json.dumps(results, indent=2), encoding="utf-8") |
| report = ( |
| f"label={label}\n" |
| f"adh={summary['adh']}/10 ref={summary['ref']}/10 len={summary['len']}/9\n" |
| ) |
| (out_dir / "report.txt").write_text(report, encoding="utf-8") |
| print("SUMMARY", summary, flush=True) |
| print("wrote", out_json, flush=True) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
