Upload q38_battery2.py with huggingface_hub

q38_battery2.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Broad 40-prompt robustness battery, auto-scored by expected substring.
+
+Greedy raw completions only (chat mode already characterized). Each item:
+(tag, prompt, [acceptable substrings — any hit = pass, case-sensitive]).
+Scores are rough (substring in 40 tokens) but identical across variants, so the
+DELTA between variants is meaningful even where the absolute bar is crude.
+
+  python q38_battery2.py --port 8991 --out q38_battery2_44.json
+"""
+import argparse, json, time, urllib.request
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--port", type=int, default=8991)
+ap.add_argument("--host", default="127.0.0.1")
+ap.add_argument("--out", required=True)
+ap.add_argument("--n", type=int, default=40)
+A = ap.parse_args()
+BASE = f"http://{A.host}:{A.port}"
+
+B = [
+    # facts: geography / science / history / culture
+    ("cap_japan",   "The capital of Japan is", ["Tokyo"]),
+    ("cap_italy",   "The capital of Italy is", ["Rome"]),
+    ("cap_egypt",   "The capital of Egypt is", ["Cairo"]),
+    ("longest_riv", "The longest river in the world is the", ["Nile", "Amazon"]),
+    ("tallest_mt",  "The tallest mountain on Earth is", ["Everest"]),
+    ("speed_light", "The speed of light is approximately", ["300,000", "3 x 10", "299", "3×10"]),
+    ("h2o",         "The chemical formula for water is", ["H2O", "H₂O"]),
+    ("dna",         "DNA stands for", ["deoxyribonucleic"]),
+    ("photosyn",    "Plants convert sunlight into energy through a process called", ["photosynthesis"]),
+    ("ww2_end",     "World War II ended in the year", ["1945"]),
+    ("moon_first",  "The first person to walk on the moon was", ["Armstrong"]),
+    ("relativity",  "The theory of relativity was developed by", ["Einstein"]),
+    ("mona_lisa",   "The Mona Lisa was painted by", ["Leonardo", "da Vinci"]),
+    ("hamlet",      "The play Hamlet was written by", ["Shakespeare"]),
+    ("planets_n",   "The number of planets in the solar system is", ["eight", "8"]),
+    ("iron_sym",    "The chemical symbol for iron is", ["Fe"]),
+    ("fr_hello",    "The French word for 'hello' is", ["bonjour", "Bonjour"]),
+    ("es_cat",      "The Spanish word for 'cat' is", ["gato"]),
+    ("jp_thanks",   "In Japanese, 'thank you' is said as", ["arigato", "arigatou", "ありがとう"]),
+    ("cont_africa", "The second largest continent by area is", ["Africa"]),
+    # arithmetic / symbolic
+    ("add3",        "7 + 5 =", ["12"]),
+    ("mult2",       "6 * 7 =", ["42"]),
+    ("precedence",  "10 + 2 * 5 =", ["20"]),
+    ("square",      "The square of 12 is", ["144"]),
+    ("half",        "Half of 90 is", ["45"]),
+    ("roman",       "The Roman numeral XIV equals", ["14"]),
+    ("binary",      "The binary number 101 in decimal is", ["5"]),
+    ("pct",         "25% of 200 is", ["50"]),
+    # code
+    ("py_len",      "In Python, to get the length of a list called items, you write", ["len(items)"]),
+    ("py_print",    "print(\"hello\" + \" \" + \"world\") outputs", ["hello world"]),
+    ("c_loop",      "for (int i = 0; i < 10; i++) { printf(\"%d\", i); } prints the numbers", ["0", "9"]),
+    ("js_arrow",    "In JavaScript, an arrow function that doubles x is written as", ["=>", "x * 2", "2 * x", "2*x", "x*2"]),
+    ("sql_count",   "To count all rows in a table called orders, the SQL is", ["COUNT", "count"]),
+    ("html_link",   "In HTML, a hyperlink to example.com is written with the tag", ["<a", "href"]),
+    ("py_dict",     "In Python, to get the value for key 'name' from dict d, you write", ["d['name']", "d[\"name\"]", ".get("]),
+    ("regex_digit", "In a regular expression, \\d matches", ["digit"]),
+    # reasoning / commonsense / continuation
+    ("bigger",      "Which is larger, a lion or a housecat? The answer is", ["lion"]),
+    ("ice",         "When water freezes it turns into", ["ice"]),
+    ("opposite_up", "The opposite of 'up' is", ["down"]),
+    ("story2",      "The old lighthouse keeper climbed the stairs one last time,", []),  # coherence only
+]
+
+
+def post(path, payload, timeout=600):
+    req = urllib.request.Request(BASE + path, json.dumps(payload).encode(),
+                                 {"Content-Type": "application/json"})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return json.load(r)
+
+
+for _ in range(300):
+    try:
+        with urllib.request.urlopen(BASE + "/health", timeout=5) as r:
+            if json.load(r).get("status") == "ok":
+                break
+    except Exception:
+        pass
+    time.sleep(2)
+else:
+    raise SystemExit("server never became healthy")
+
+results, passed, scored = {}, 0, 0
+for tag, p, expect in B:
+    try:
+        r = post("/completion", {"prompt": p, "n_predict": A.n, "temperature": 0.0,
+                                 "top_k": 1, "top_p": 1.0, "seed": 7})
+        txt = r.get("content", "")
+    except Exception as e:
+        # a model degraded enough to emit invalid UTF-8 makes the server 500;
+        # that is a miss, not a reason to abandon the run
+        txt = f"<request failed: {type(e).__name__}>"
+
+    ok = None
+    if expect:
+        scored += 1
+        ok = any(e in txt for e in expect)
+        passed += ok
+    results[tag] = {"prompt": p, "completion": txt, "pass": ok}
+    print(f"{tag:12} {'PASS' if ok else 'FAIL' if ok is not None else '----'}  "
+          f"{txt[:80].replace(chr(10),' ')}", flush=True)
+
+out = {"score": f"{passed}/{scored}", "results": results,
+       "ts": time.strftime("%Y-%m-%d %H:%M")}
+json.dump(out, open(A.out, "w"), indent=1)
+print(f"\nSCORE {passed}/{scored}  -> {A.out}")