Buckets:
| """Benchmark / training task builders. Every task yields (state, questions, gold) triples. | |
| Questions are always written in English (the developer's schema); the *state* is in the | |
| target language, which is how System One is used in production. | |
| """ | |
| import json | |
| import random | |
| from collections import defaultdict | |
| D = "./data" | |
| SCEN_DESC = { | |
| "alarm": "alarms and wake-up times", "audio": "device volume and sound", "iot": "smart home devices, lights, cleaning", | |
| "calendar": "meetings, reminders, calendar events", "play": "play music, podcasts, radio, audiobooks, games", | |
| "general": "jokes, greetings, chit-chat, repeat, confirm", "datetime": "date and time questions", | |
| "takeaway": "food delivery and takeaway orders", "news": "news headlines and updates", | |
| "music": "music preferences, song info, settings", "weather": "weather forecast", | |
| "qa": "factual questions, definitions, maths, currency", "social": "social media posts and complaints", | |
| "recommendation": "recommendations for places, events, movies", "cooking": "recipes and cooking", | |
| "transport": "tickets, taxis, trains, traffic", "email": "reading, sending and managing email", | |
| "lists": "to-do and shopping lists", | |
| } | |
| SIB_DESC = { | |
| "science/technology": "science, research, technology", "travel": "travel and tourism", | |
| "politics": "politics, government, elections", "sports": "sports and athletes", | |
| "health": "health, medicine, disease", "entertainment": "films, music, celebrities, culture", | |
| "geography": "places, geography, nature", | |
| } | |
| def _jl(name): | |
| return [json.loads(l) for l in open(f"{D}/{name}.jsonl")] | |
| def human(label): | |
| return label.replace("_", " ").replace("iot", "smart home").replace("qa", "question") | |
| def massive_scenario(lang, split, n=None, seed=0): | |
| rows = _jl(f"massive_{lang}_{split}") | |
| random.Random(seed).shuffle(rows) | |
| q = {"scenario": {"type": "choice", "instructions": "Which domain does the user's request to the voice assistant belong to?", | |
| "criteria": SCEN_DESC}} | |
| return [(r["text"], q, {"scenario": r["label"].split("_")[0]}) for r in rows[:n]] | |
| def massive_intent(lang, split, n=None, k=20, seed=0): | |
| rows = _jl(f"massive_{lang}_{split}") | |
| labels = sorted({r["label"] for r in _jl(f"massive_en_train")}) | |
| rng = random.Random(seed) | |
| rng.shuffle(rows) | |
| out = [] | |
| for r in rows[:n]: | |
| opts = rng.sample([l for l in labels if l != r["label"]], k - 1) + [r["label"]] | |
| rng.shuffle(opts) | |
| q = {"intent": {"type": "choice", "instructions": "What does the user want the voice assistant to do?", | |
| "criteria": {o: human(o) for o in opts}}} | |
| out.append((r["text"], q, {"intent": r["label"]})) | |
| return out | |
| def intent_noul(lang, split, n=None, seed=0): | |
| """Balanced yes/no: 'Does the user want to <intent>?' -- half true intent, half a random other.""" | |
| rows = _jl(f"massive_{lang}_{split}") | |
| labels = sorted({r["label"] for r in _jl(f"massive_en_train")}) | |
| rng = random.Random(seed + 7) | |
| rng.shuffle(rows) | |
| out = [] | |
| for i, r in enumerate(rows[:n]): | |
| pos = i % 2 == 0 | |
| lab = r["label"] if pos else rng.choice([l for l in labels if l != r["label"]]) | |
| q = {"match": {"type": "noul", "instructions": "Is the user asking the assistant to: %s?" % human(lab)}} | |
| out.append((r["text"], q, {"match": pos})) | |
| return out | |
| def sib(lang, split, n=None, seed=0): | |
| cfg = {"az": "azj_Latn", "en": "eng_Latn"}[lang] | |
| rows = _jl(f"sib_{cfg}_{split}") | |
| random.Random(seed).shuffle(rows) | |
| q = {"topic": {"type": "choice", "instructions": "What is the topic of this text?", "criteria": SIB_DESC}} | |
| return [(r["text"], q, {"topic": r["category"]}) for r in rows[:n]] | |
| def az_sentiment(split, n=1000, seed=0, as_noul=False): | |
| rows = [r for r in _jl(f"azsent_{split}") if r["score"] in (1, 2, 5) and len(r["content"] or "") >= 12] | |
| rng = random.Random(seed) | |
| neg = [r for r in rows if r["score"] <= 2] | |
| pos = [r for r in rows if r["score"] == 5] | |
| rng.shuffle(neg); rng.shuffle(pos) | |
| sel = neg[: n // 2] + pos[: n // 2] | |
| rng.shuffle(sel) | |
| if as_noul: | |
| q = {"negative": {"type": "noul", "instructions": "Is this app review negative or a complaint?"}} | |
| return [(r["content"], q, {"negative": r["score"] <= 2}) for r in sel] | |
| q = {"sentiment": {"type": "choice", "instructions": "What is the sentiment of this app review?", | |
| "criteria": {"positive": "satisfied, praise", "negative": "unhappy, complaint, problem"}}} | |
| return [(r["content"], q, {"sentiment": "negative" if r["score"] <= 2 else "positive"}) for r in sel] | |
| def eval_suite(n=1000): | |
| """name -> list of triples. Test splits only.""" | |
| s = {} | |
| for lang in ("en", "az"): | |
| s[f"MASSIVE-scenario ({lang})"] = massive_scenario(lang, "test", n) | |
| s[f"MASSIVE-intent@20 ({lang})"] = massive_intent(lang, "test", n) | |
| s[f"Intent yes/no ({lang})"] = intent_noul(lang, "test", n) | |
| s[f"SIB-200 topic ({lang})"] = sib(lang, "test") | |
| s["AZ app-review sentiment"] = az_sentiment("test", n) | |
| return s | |
| # ------------------------------------------------------------------ multilingual + Jev-comparison + routing | |
| SIB_CFG = {"az": "azj_Latn", "en": "eng_Latn", "ru": "rus_Cyrl", "tr": "tur_Latn", "de": "deu_Latn", "ar": "arb_Arab", | |
| "hi": "hin_Deva", "zh": "zho_Hans", "kk": "kaz_Cyrl", "uz": "uzn_Latn"} | |
| def sib_any(lang, split, n=None, seed=0): | |
| rows = _jl(f"sib_{SIB_CFG[lang]}_{split}") | |
| random.Random(seed).shuffle(rows) | |
| q = {"topic": {"type": "choice", "instructions": "What is the topic of this text?", "criteria": SIB_DESC}} | |
| return [(r["text"], q, {"topic": r["category"]}) for r in rows[:n]] | |
| def _balanced(rows, key, per, seed): | |
| rng = random.Random(seed) | |
| by = defaultdict(list) | |
| for r in rows: | |
| by[r[key]].append(r) | |
| out = [] | |
| for k in sorted(by): | |
| rng.shuffle(by[k]); out += by[k][:per] | |
| rng.shuffle(out) | |
| return out | |
| def agnews(per=250, seed=20260917): | |
| names = ["World", "Sports", "Business", "Sci/Tech"] | |
| desc = {"world": "world news, politics, international affairs", "sports": "sports", "business": "business, economy, companies, markets", | |
| "sci_tech": "science and technology"} | |
| q = {"topic": {"type": "choice", "instructions": "What is the topic of this news article?", "criteria": desc}} | |
| keys = list(desc) | |
| return [(r["text"], q, {"topic": keys[r["label"]]}) for r in _balanced(_jl("agnews"), "label", per, seed)] | |
| def emotion(per=200, seed=20260917): | |
| names = ["sadness", "joy", "love", "anger", "fear", "surprise"] | |
| # label descriptions (as in the published Jev protocol); this wording was picked on the DAIR *validation* split | |
| desc = {"sadness": "sad, unhappy, hurt, lonely, disappointed", "joy": "happy, glad, excited, grateful, content", | |
| "love": "love, affection, caring, romantic, fond", "anger": "angry, annoyed, irritated, hostile, resentful", | |
| "fear": "afraid, scared, anxious, nervous, worried", "surprise": "surprised, amazed, shocked, astonished"} | |
| q = {"emotion": {"type": "choice", "instructions": "Which emotion does the writer express?", "criteria": desc}} | |
| return [(r["text"], q, {"emotion": names[r["label"]]}) for r in _balanced(_jl("emotion"), "label", per, seed)] | |
| def smsspam(per=500, seed=20260917): | |
| # choice form picked on held-out SMS messages (not the test sample) | |
| q = {"spam": {"type": "choice", "instructions": "Is this SMS spam or a normal message (ham)?", | |
| "criteria": {"ham": "a normal personal or legitimate message", | |
| "spam": "unsolicited advertising, prize, lottery, premium-rate offer or scam"}}} | |
| return [(r["sms"], q, {"spam": "spam" if r["label"] == 1 else "ham"}) for r in _balanced(_jl("smsspam"), "label", per, seed)] | |
| def banking77(per=13, seed=20260917): | |
| rows = _balanced(_jl("banking77"), "label_text", per, seed) | |
| labels = sorted({r["label_text"] for r in _jl("banking77")}) | |
| q = {"intent": {"type": "choice", "instructions": "What is the banking customer asking about?", | |
| "criteria": {l: l.replace("_", " ") for l in labels}}} | |
| return [(r["text"], q, {"intent": r["label_text"]}) for r in rows] | |
| LLM_ZOO = { | |
| "legolas-flash": "legal model: laws, articles of codes, courts, contracts, rights, penalties", | |
| "dragonfly-1.5": "everyday tasks model: chat, letters, lists, planning, advice, translation", | |
| "wahoo-1.5": "fast model: short simple questions with a quick one-line answer", | |
| "gpt-5.6-luna": "strongest reasoning model: hard math, science, programming, deep multi-step analysis", | |
| } | |
| # Hand-written (not LLM-generated) held-out routing test. | |
| LLM_ROUTE = [ | |
| ("511 maddə nə tənzimləyir", "legolas-flash"), ("Cinayət Məcəlləsinin 179-cu maddəsi üzrə cəza nədir?", "legolas-flash"), | |
| ("Əmək müqaviləsini işəgötürən birtərəfli poza bilərmi?", "legolas-flash"), ("Boşanma zamanı əmlak necə bölünür?", "legolas-flash"), | |
| ("Kirayə müqaviləsində girov pulunu geri qaytarmırlar, nə edim?", "legolas-flash"), ("Vergi Məcəlləsinə görə ƏDV dərəcəsi neçədir?", "legolas-flash"), | |
| ("inzibati xetalar mecellesi 327 madde", "legolas-flash"), ("Miras hüququ ilə bağlı vəsiyyətnamə necə tərtib olunur?", "legolas-flash"), | |
| ("Məhkəməyə iddia ərizəsi necə yazılır?", "legolas-flash"), ("What does article 511 of the civil code regulate?", "legolas-flash"), | |
| ("Что регулирует статья 511 Гражданского кодекса?", "legolas-flash"), ("Patent hüququnu necə qoruya bilərəm?", "legolas-flash"), | |
| ("Salam, necəsən? Sabah üçün alış-veriş siyahısı yaz", "dragonfly-1.5"), ("Müdirimə məzuniyyət üçün məktub yaz", "dragonfly-1.5"), | |
| ("Həftəsonu Bakıda nə edə bilərəm?", "dragonfly-1.5"), ("Anamın ad günü üçün təbrik mesajı yaz", "dragonfly-1.5"), | |
| ("bu metni ingiliscəyə tərcümə et: sabah görüşək", "dragonfly-1.5"), ("Səhər yeməyi üçün sağlam reseptlər təklif et", "dragonfly-1.5"), | |
| ("Mənə bu həftə üçün idman planı qur", "dragonfly-1.5"), ("Write a friendly email inviting my team to lunch", "dragonfly-1.5"), | |
| ("Помоги составить список покупок на неделю", "dragonfly-1.5"), ("Uşağım üçün yuxu nağılı danış", "dragonfly-1.5"), | |
| ("Azərbaycanın paytaxtı haradır?", "wahoo-1.5"), ("Tez cavab ver: 2+2 neçədir?", "wahoo-1.5"), | |
| ("Bu gün həftənin hansı günüdür?", "wahoo-1.5"), ("Bir kilometr neçə metrdir?", "wahoo-1.5"), | |
| ("Suyun qaynama temperaturu?", "wahoo-1.5"), ("What is the capital of France?", "wahoo-1.5"), | |
| ("Сколько будет 7 умножить на 8?", "wahoo-1.5"), ("Xəzər dənizi dənizdir yoxsa göl? qısa cavab", "wahoo-1.5"), | |
| ("100 dollar neçə manatdır təxminən?", "wahoo-1.5"), ("Nizami Gəncəvi kimdir? bir cümlə", "wahoo-1.5"), | |
| ("Kvant fizikasında dolaşıqlığı ətraflı izah et və riyazi sübut ver", "gpt-5.6-luna"), | |
| ("Mənə python kodunda bu xətanı tap və optimallaşdır, çoxaddımlı analiz lazımdır", "gpt-5.6-luna"), | |
| ("Riman hipotezinin mahiyyətini və nəticələrini dərindən analiz et", "gpt-5.6-luna"), | |
| ("Neft qiymətlərinin Azərbaycan büdcəsinə təsirini ekonometrik model ilə təhlil et", "gpt-5.6-luna"), | |
| ("Paylanmış sistemdə konsensus alqoritmi dizayn et və Raft ilə müqayisə et", "gpt-5.6-luna"), | |
| ("Prove that there are infinitely many primes and generalize to arithmetic progressions", "gpt-5.6-luna"), | |
| ("Спроектируй архитектуру микросервисов для банка с учётом отказоустойчивости", "gpt-5.6-luna"), | |
| ("Bu diferensial tənliyi həll et və həllin dayanıqlığını araşdır: y'' + 3y' + 2y = sin(x)", "gpt-5.6-luna"), | |
| ("React tətbiqimdə yaddaş sızmasını tap, kod aşağıdadır, addım-addım təhlil et", "gpt-5.6-luna"), | |
| ("İqlim dəyişikliyinin Xəzər ekosisteminə uzunmüddətli təsirlərini çoxtərəfli təhlil et", "gpt-5.6-luna"), | |
| ] | |
| def llm_route(): | |
| q = {"model": {"type": "choice", "instructions": "Which AI model should answer this user request?", "criteria": LLM_ZOO}} | |
| return [(s, q, {"model": g}) for s, g in LLM_ROUTE] | |
| def synth_test(kind, n=None): | |
| path = f"{D}/synth/{kind}_test.jsonl" | |
| rows = [json.loads(l) for l in open(path)][:n] | |
| out = [] | |
| for r in rows: | |
| for qid, g in r["gold"].items(): | |
| out.append((r["state"], {qid: r["questions"][qid]}, {qid: g})) | |
| return out | |
| MASSIVE_LANGS = ["en", "az", "ru", "tr", "de", "fr", "es", "ar", "hi", "zh-CN", "fa", "ka", "ja", "ko"] | |
| def eval_suite_v2(n=500): | |
| s = {} | |
| s["LLM routing (hand-written)"] = llm_route() | |
| for kind in ("routing", "decisions", "guard", "taxonomy", "emotion"): | |
| try: | |
| s[f"Synthetic {kind} (held-out schemas)"] = synth_test(kind, 1500) | |
| except FileNotFoundError: | |
| pass | |
| s["AG News (4)"] = agnews() | |
| s["DAIR Emotion (6)"] = emotion() | |
| s["Banking77 (77)"] = banking77() | |
| s["SMS spam (yes/no)"] = smsspam() | |
| s["AZ app-review sentiment"] = az_sentiment("test", 1000) | |
| for l in ("az", "en"): | |
| s[f"MASSIVE-intent@20 ({l})"] = massive_intent(l, "test", 1000) | |
| s[f"Intent yes/no ({l})"] = intent_noul(l, "test", 1000) | |
| for l in MASSIVE_LANGS: | |
| s[f"MASSIVE-scenario ({l})"] = massive_scenario(l, "test", n) | |
| for l in SIB_CFG: | |
| s[f"SIB-200 ({l})"] = sib_any(l, "test") | |
| return s | |
Xet Storage Details
- Size:
- 13.7 kB
- Xet hash:
- 23a2cdf26ddb80a2b125806695c7a619671eb3a831903e4f87a55cf76e093f56
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.