"""Calibration requests in Prism's own input format, built from multilingual Wikipedia passages (11 languages). Disjoint from the evaluation suite (different texts and question wording).""" import json import random LANG_NAMES = {"en": "English", "hi": "Hindi", "bn": "Bengali", "ta": "Tamil", "te": "Telugu", "mr": "Marathi", "gu": "Gujarati", "kn": "Kannada", "ml": "Malayalam", "or": "Odia", "pa": "Punjabi"} QUESTIONS = [ lambda r, l: {"type": "choice", "instructions": "What is the main subject area of this text?", "criteria": {"history": "history and past events", "science": "science and technology", "geography": "places and geography", "people": "a biography of a person", "culture": "arts, religion, culture", "politics": "government and politics", "sports": None, "other": None}}, lambda r, l: {"type": "choice", "instructions": "In which language is the passage written?", "criteria": dict(r.sample(sorted(LANG_NAMES.items()), 4) + [(l, LANG_NAMES[l])])}, lambda r, l: {"type": "yes_no", "instructions": "Does the passage mention a specific year or date?"}, lambda r, l: {"type": "yes_no", "instructions": "Is this passage about a living person?"}, lambda r, l: {"type": "yes_no", "instructions": "Would this text be suitable for a children's encyclopedia?"}, lambda r, l: {"type": "yes_no", "instructions": "Does the text contain a factual error?"}, lambda r, l: {"type": "score", "instructions": "How technical is the language of this passage?", "criteria": ["very simple", "simple", "moderate", "technical", "highly specialised"]}, lambda r, l: {"type": "score", "instructions": "How well written is this text?", "criteria": {"poor": "many errors", "fair": "some problems", "good": "clear", "great": "excellent prose"}}, lambda r, l: {"type": "score", "instructions": "How likely is a general reader to find this interesting?", "criteria": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]}, lambda r, l: {"type": "choice", "instructions": "Which audience is this text written for?", "criteria": {"general": "general public", "student": "students", "expert": "domain experts"}}, lambda r, l: {"type": "choice", "instructions": "What tone does the passage have?", "criteria": {"neutral": None, "positive": None, "negative": None, "promotional": "advertising"}}, lambda r, l: {"type": "yes_no", "instructions": "किया यह पाठ किसी स्थान के बारे में है?" if l != "en" else "Is this text about a place?"}, lambda r, l: {"type": "choice", "instructions": "If a user submitted this as a support ticket, which queue fits best?", "criteria": {"billing": "payments and refunds", "tech": "technical issue", "spam": "not a real ticket", "general": "general enquiry"}}, lambda r, l: {"type": "score", "instructions": "How urgent would it be to act on this information?", "criteria": ["not urgent", "low", "medium", "high", "critical"]}, ] def make_requests(wiki_path, seed=0, n_per_lang=400): wiki = json.load(open(wiki_path)) r = random.Random(seed) reqs = [] for lang, texts in wiki.items(): for t in texts[:n_per_lang]: u = r.random() n_chars = r.randint(150, 900) if u < 0.55 else (r.randint(900, 3500) if u < 0.95 else r.randint(3500, 12000)) start = r.randint(0, max(0, len(t) - n_chars)) if len(t) > n_chars else 0 passage = t[start:start + n_chars] if r.random() < 0.25: state = {"source": "encyclopedia", "lang": lang, "text": passage} else: state = passage qs = {} for i, qi in enumerate(r.sample(range(len(QUESTIONS)), r.randint(1, 4))): qs[f"q{i}"] = QUESTIONS[qi](r, lang) reqs.append((state, qs)) r.shuffle(reqs) return reqs