Geoo's picture
download
raw
13.7 kB
"""Benchmark / training task builders. Every task yields (state, questions, gold) triples.
Questions are always written in English (the developer's schema); the *state* is in the
target language, which is how System One is used in production.
"""
import json
import random
from collections import defaultdict
D = "./data"
SCEN_DESC = {
"alarm": "alarms and wake-up times", "audio": "device volume and sound", "iot": "smart home devices, lights, cleaning",
"calendar": "meetings, reminders, calendar events", "play": "play music, podcasts, radio, audiobooks, games",
"general": "jokes, greetings, chit-chat, repeat, confirm", "datetime": "date and time questions",
"takeaway": "food delivery and takeaway orders", "news": "news headlines and updates",
"music": "music preferences, song info, settings", "weather": "weather forecast",
"qa": "factual questions, definitions, maths, currency", "social": "social media posts and complaints",
"recommendation": "recommendations for places, events, movies", "cooking": "recipes and cooking",
"transport": "tickets, taxis, trains, traffic", "email": "reading, sending and managing email",
"lists": "to-do and shopping lists",
}
SIB_DESC = {
"science/technology": "science, research, technology", "travel": "travel and tourism",
"politics": "politics, government, elections", "sports": "sports and athletes",
"health": "health, medicine, disease", "entertainment": "films, music, celebrities, culture",
"geography": "places, geography, nature",
}
def _jl(name):
return [json.loads(l) for l in open(f"{D}/{name}.jsonl")]
def human(label):
return label.replace("_", " ").replace("iot", "smart home").replace("qa", "question")
def massive_scenario(lang, split, n=None, seed=0):
rows = _jl(f"massive_{lang}_{split}")
random.Random(seed).shuffle(rows)
q = {"scenario": {"type": "choice", "instructions": "Which domain does the user's request to the voice assistant belong to?",
"criteria": SCEN_DESC}}
return [(r["text"], q, {"scenario": r["label"].split("_")[0]}) for r in rows[:n]]
def massive_intent(lang, split, n=None, k=20, seed=0):
rows = _jl(f"massive_{lang}_{split}")
labels = sorted({r["label"] for r in _jl(f"massive_en_train")})
rng = random.Random(seed)
rng.shuffle(rows)
out = []
for r in rows[:n]:
opts = rng.sample([l for l in labels if l != r["label"]], k - 1) + [r["label"]]
rng.shuffle(opts)
q = {"intent": {"type": "choice", "instructions": "What does the user want the voice assistant to do?",
"criteria": {o: human(o) for o in opts}}}
out.append((r["text"], q, {"intent": r["label"]}))
return out
def intent_noul(lang, split, n=None, seed=0):
"""Balanced yes/no: 'Does the user want to <intent>?' -- half true intent, half a random other."""
rows = _jl(f"massive_{lang}_{split}")
labels = sorted({r["label"] for r in _jl(f"massive_en_train")})
rng = random.Random(seed + 7)
rng.shuffle(rows)
out = []
for i, r in enumerate(rows[:n]):
pos = i % 2 == 0
lab = r["label"] if pos else rng.choice([l for l in labels if l != r["label"]])
q = {"match": {"type": "noul", "instructions": "Is the user asking the assistant to: %s?" % human(lab)}}
out.append((r["text"], q, {"match": pos}))
return out
def sib(lang, split, n=None, seed=0):
cfg = {"az": "azj_Latn", "en": "eng_Latn"}[lang]
rows = _jl(f"sib_{cfg}_{split}")
random.Random(seed).shuffle(rows)
q = {"topic": {"type": "choice", "instructions": "What is the topic of this text?", "criteria": SIB_DESC}}
return [(r["text"], q, {"topic": r["category"]}) for r in rows[:n]]
def az_sentiment(split, n=1000, seed=0, as_noul=False):
rows = [r for r in _jl(f"azsent_{split}") if r["score"] in (1, 2, 5) and len(r["content"] or "") >= 12]
rng = random.Random(seed)
neg = [r for r in rows if r["score"] <= 2]
pos = [r for r in rows if r["score"] == 5]
rng.shuffle(neg); rng.shuffle(pos)
sel = neg[: n // 2] + pos[: n // 2]
rng.shuffle(sel)
if as_noul:
q = {"negative": {"type": "noul", "instructions": "Is this app review negative or a complaint?"}}
return [(r["content"], q, {"negative": r["score"] <= 2}) for r in sel]
q = {"sentiment": {"type": "choice", "instructions": "What is the sentiment of this app review?",
"criteria": {"positive": "satisfied, praise", "negative": "unhappy, complaint, problem"}}}
return [(r["content"], q, {"sentiment": "negative" if r["score"] <= 2 else "positive"}) for r in sel]
def eval_suite(n=1000):
"""name -> list of triples. Test splits only."""
s = {}
for lang in ("en", "az"):
s[f"MASSIVE-scenario ({lang})"] = massive_scenario(lang, "test", n)
s[f"MASSIVE-intent@20 ({lang})"] = massive_intent(lang, "test", n)
s[f"Intent yes/no ({lang})"] = intent_noul(lang, "test", n)
s[f"SIB-200 topic ({lang})"] = sib(lang, "test")
s["AZ app-review sentiment"] = az_sentiment("test", n)
return s
# ------------------------------------------------------------------ multilingual + Jev-comparison + routing
SIB_CFG = {"az": "azj_Latn", "en": "eng_Latn", "ru": "rus_Cyrl", "tr": "tur_Latn", "de": "deu_Latn", "ar": "arb_Arab",
"hi": "hin_Deva", "zh": "zho_Hans", "kk": "kaz_Cyrl", "uz": "uzn_Latn"}
def sib_any(lang, split, n=None, seed=0):
rows = _jl(f"sib_{SIB_CFG[lang]}_{split}")
random.Random(seed).shuffle(rows)
q = {"topic": {"type": "choice", "instructions": "What is the topic of this text?", "criteria": SIB_DESC}}
return [(r["text"], q, {"topic": r["category"]}) for r in rows[:n]]
def _balanced(rows, key, per, seed):
rng = random.Random(seed)
by = defaultdict(list)
for r in rows:
by[r[key]].append(r)
out = []
for k in sorted(by):
rng.shuffle(by[k]); out += by[k][:per]
rng.shuffle(out)
return out
def agnews(per=250, seed=20260917):
names = ["World", "Sports", "Business", "Sci/Tech"]
desc = {"world": "world news, politics, international affairs", "sports": "sports", "business": "business, economy, companies, markets",
"sci_tech": "science and technology"}
q = {"topic": {"type": "choice", "instructions": "What is the topic of this news article?", "criteria": desc}}
keys = list(desc)
return [(r["text"], q, {"topic": keys[r["label"]]}) for r in _balanced(_jl("agnews"), "label", per, seed)]
def emotion(per=200, seed=20260917):
names = ["sadness", "joy", "love", "anger", "fear", "surprise"]
# label descriptions (as in the published Jev protocol); this wording was picked on the DAIR *validation* split
desc = {"sadness": "sad, unhappy, hurt, lonely, disappointed", "joy": "happy, glad, excited, grateful, content",
"love": "love, affection, caring, romantic, fond", "anger": "angry, annoyed, irritated, hostile, resentful",
"fear": "afraid, scared, anxious, nervous, worried", "surprise": "surprised, amazed, shocked, astonished"}
q = {"emotion": {"type": "choice", "instructions": "Which emotion does the writer express?", "criteria": desc}}
return [(r["text"], q, {"emotion": names[r["label"]]}) for r in _balanced(_jl("emotion"), "label", per, seed)]
def smsspam(per=500, seed=20260917):
# choice form picked on held-out SMS messages (not the test sample)
q = {"spam": {"type": "choice", "instructions": "Is this SMS spam or a normal message (ham)?",
"criteria": {"ham": "a normal personal or legitimate message",
"spam": "unsolicited advertising, prize, lottery, premium-rate offer or scam"}}}
return [(r["sms"], q, {"spam": "spam" if r["label"] == 1 else "ham"}) for r in _balanced(_jl("smsspam"), "label", per, seed)]
def banking77(per=13, seed=20260917):
rows = _balanced(_jl("banking77"), "label_text", per, seed)
labels = sorted({r["label_text"] for r in _jl("banking77")})
q = {"intent": {"type": "choice", "instructions": "What is the banking customer asking about?",
"criteria": {l: l.replace("_", " ") for l in labels}}}
return [(r["text"], q, {"intent": r["label_text"]}) for r in rows]
LLM_ZOO = {
"legolas-flash": "legal model: laws, articles of codes, courts, contracts, rights, penalties",
"dragonfly-1.5": "everyday tasks model: chat, letters, lists, planning, advice, translation",
"wahoo-1.5": "fast model: short simple questions with a quick one-line answer",
"gpt-5.6-luna": "strongest reasoning model: hard math, science, programming, deep multi-step analysis",
}
# Hand-written (not LLM-generated) held-out routing test.
LLM_ROUTE = [
("511 maddə nə tənzimləyir", "legolas-flash"), ("Cinayət Məcəlləsinin 179-cu maddəsi üzrə cəza nədir?", "legolas-flash"),
("Əmək müqaviləsini işəgötürən birtərəfli poza bilərmi?", "legolas-flash"), ("Boşanma zamanı əmlak necə bölünür?", "legolas-flash"),
("Kirayə müqaviləsində girov pulunu geri qaytarmırlar, nə edim?", "legolas-flash"), ("Vergi Məcəlləsinə görə ƏDV dərəcəsi neçədir?", "legolas-flash"),
("inzibati xetalar mecellesi 327 madde", "legolas-flash"), ("Miras hüququ ilə bağlı vəsiyyətnamə necə tərtib olunur?", "legolas-flash"),
("Məhkəməyə iddia ərizəsi necə yazılır?", "legolas-flash"), ("What does article 511 of the civil code regulate?", "legolas-flash"),
("Что регулирует статья 511 Гражданского кодекса?", "legolas-flash"), ("Patent hüququnu necə qoruya bilərəm?", "legolas-flash"),
("Salam, necəsən? Sabah üçün alış-veriş siyahısı yaz", "dragonfly-1.5"), ("Müdirimə məzuniyyət üçün məktub yaz", "dragonfly-1.5"),
("Həftəsonu Bakıda nə edə bilərəm?", "dragonfly-1.5"), ("Anamın ad günü üçün təbrik mesajı yaz", "dragonfly-1.5"),
("bu metni ingiliscəyə tərcümə et: sabah görüşək", "dragonfly-1.5"), ("Səhər yeməyi üçün sağlam reseptlər təklif et", "dragonfly-1.5"),
("Mənə bu həftə üçün idman planı qur", "dragonfly-1.5"), ("Write a friendly email inviting my team to lunch", "dragonfly-1.5"),
("Помоги составить список покупок на неделю", "dragonfly-1.5"), ("Uşağım üçün yuxu nağılı danış", "dragonfly-1.5"),
("Azərbaycanın paytaxtı haradır?", "wahoo-1.5"), ("Tez cavab ver: 2+2 neçədir?", "wahoo-1.5"),
("Bu gün həftənin hansı günüdür?", "wahoo-1.5"), ("Bir kilometr neçə metrdir?", "wahoo-1.5"),
("Suyun qaynama temperaturu?", "wahoo-1.5"), ("What is the capital of France?", "wahoo-1.5"),
("Сколько будет 7 умножить на 8?", "wahoo-1.5"), ("Xəzər dənizi dənizdir yoxsa göl? qısa cavab", "wahoo-1.5"),
("100 dollar neçə manatdır təxminən?", "wahoo-1.5"), ("Nizami Gəncəvi kimdir? bir cümlə", "wahoo-1.5"),
("Kvant fizikasında dolaşıqlığı ətraflı izah et və riyazi sübut ver", "gpt-5.6-luna"),
("Mənə python kodunda bu xətanı tap və optimallaşdır, çoxaddımlı analiz lazımdır", "gpt-5.6-luna"),
("Riman hipotezinin mahiyyətini və nəticələrini dərindən analiz et", "gpt-5.6-luna"),
("Neft qiymətlərinin Azərbaycan büdcəsinə təsirini ekonometrik model ilə təhlil et", "gpt-5.6-luna"),
("Paylanmış sistemdə konsensus alqoritmi dizayn et və Raft ilə müqayisə et", "gpt-5.6-luna"),
("Prove that there are infinitely many primes and generalize to arithmetic progressions", "gpt-5.6-luna"),
("Спроектируй архитектуру микросервисов для банка с учётом отказоустойчивости", "gpt-5.6-luna"),
("Bu diferensial tənliyi həll et və həllin dayanıqlığını araşdır: y'' + 3y' + 2y = sin(x)", "gpt-5.6-luna"),
("React tətbiqimdə yaddaş sızmasını tap, kod aşağıdadır, addım-addım təhlil et", "gpt-5.6-luna"),
("İqlim dəyişikliyinin Xəzər ekosisteminə uzunmüddətli təsirlərini çoxtərəfli təhlil et", "gpt-5.6-luna"),
]
def llm_route():
q = {"model": {"type": "choice", "instructions": "Which AI model should answer this user request?", "criteria": LLM_ZOO}}
return [(s, q, {"model": g}) for s, g in LLM_ROUTE]
def synth_test(kind, n=None):
path = f"{D}/synth/{kind}_test.jsonl"
rows = [json.loads(l) for l in open(path)][:n]
out = []
for r in rows:
for qid, g in r["gold"].items():
out.append((r["state"], {qid: r["questions"][qid]}, {qid: g}))
return out
MASSIVE_LANGS = ["en", "az", "ru", "tr", "de", "fr", "es", "ar", "hi", "zh-CN", "fa", "ka", "ja", "ko"]
def eval_suite_v2(n=500):
s = {}
s["LLM routing (hand-written)"] = llm_route()
for kind in ("routing", "decisions", "guard", "taxonomy", "emotion"):
try:
s[f"Synthetic {kind} (held-out schemas)"] = synth_test(kind, 1500)
except FileNotFoundError:
pass
s["AG News (4)"] = agnews()
s["DAIR Emotion (6)"] = emotion()
s["Banking77 (77)"] = banking77()
s["SMS spam (yes/no)"] = smsspam()
s["AZ app-review sentiment"] = az_sentiment("test", 1000)
for l in ("az", "en"):
s[f"MASSIVE-intent@20 ({l})"] = massive_intent(l, "test", 1000)
s[f"Intent yes/no ({l})"] = intent_noul(l, "test", 1000)
for l in MASSIVE_LANGS:
s[f"MASSIVE-scenario ({l})"] = massive_scenario(l, "test", n)
for l in SIB_CFG:
s[f"SIB-200 ({l})"] = sib_any(l, "test")
return s

Xet Storage Details

Size:
13.7 kB
·
Xet hash:
23a2cdf26ddb80a2b125806695c7a619671eb3a831903e4f87a55cf76e093f56

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.