| """Calibration requests in Prism's own input format, built from multilingual Wikipedia passages (11 languages). |
| Disjoint from the evaluation suite (different texts and question wording).""" |
| import json |
| import random |
|
|
| LANG_NAMES = {"en": "English", "hi": "Hindi", "bn": "Bengali", "ta": "Tamil", "te": "Telugu", "mr": "Marathi", |
| "gu": "Gujarati", "kn": "Kannada", "ml": "Malayalam", "or": "Odia", "pa": "Punjabi"} |
|
|
| QUESTIONS = [ |
| lambda r, l: {"type": "choice", "instructions": "What is the main subject area of this text?", |
| "criteria": {"history": "history and past events", "science": "science and technology", |
| "geography": "places and geography", "people": "a biography of a person", |
| "culture": "arts, religion, culture", "politics": "government and politics", |
| "sports": None, "other": None}}, |
| lambda r, l: {"type": "choice", "instructions": "In which language is the passage written?", |
| "criteria": dict(r.sample(sorted(LANG_NAMES.items()), 4) + [(l, LANG_NAMES[l])])}, |
| lambda r, l: {"type": "yes_no", "instructions": "Does the passage mention a specific year or date?"}, |
| lambda r, l: {"type": "yes_no", "instructions": "Is this passage about a living person?"}, |
| lambda r, l: {"type": "yes_no", "instructions": "Would this text be suitable for a children's encyclopedia?"}, |
| lambda r, l: {"type": "yes_no", "instructions": "Does the text contain a factual error?"}, |
| lambda r, l: {"type": "score", "instructions": "How technical is the language of this passage?", |
| "criteria": ["very simple", "simple", "moderate", "technical", "highly specialised"]}, |
| lambda r, l: {"type": "score", "instructions": "How well written is this text?", |
| "criteria": {"poor": "many errors", "fair": "some problems", "good": "clear", "great": "excellent prose"}}, |
| lambda r, l: {"type": "score", "instructions": "How likely is a general reader to find this interesting?", |
| "criteria": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]}, |
| lambda r, l: {"type": "choice", "instructions": "Which audience is this text written for?", |
| "criteria": {"general": "general public", "student": "students", "expert": "domain experts"}}, |
| lambda r, l: {"type": "choice", "instructions": "What tone does the passage have?", |
| "criteria": {"neutral": None, "positive": None, "negative": None, "promotional": "advertising"}}, |
| lambda r, l: {"type": "yes_no", "instructions": "किया यह पाठ किसी स्थान के बारे में है?" if l != "en" else |
| "Is this text about a place?"}, |
| lambda r, l: {"type": "choice", "instructions": "If a user submitted this as a support ticket, which queue fits best?", |
| "criteria": {"billing": "payments and refunds", "tech": "technical issue", "spam": "not a real ticket", |
| "general": "general enquiry"}}, |
| lambda r, l: {"type": "score", "instructions": "How urgent would it be to act on this information?", |
| "criteria": ["not urgent", "low", "medium", "high", "critical"]}, |
| ] |
|
|
|
|
| def make_requests(wiki_path, seed=0, n_per_lang=400): |
| wiki = json.load(open(wiki_path)) |
| r = random.Random(seed) |
| reqs = [] |
| for lang, texts in wiki.items(): |
| for t in texts[:n_per_lang]: |
| u = r.random() |
| n_chars = r.randint(150, 900) if u < 0.55 else (r.randint(900, 3500) if u < 0.95 else r.randint(3500, 12000)) |
| start = r.randint(0, max(0, len(t) - n_chars)) if len(t) > n_chars else 0 |
| passage = t[start:start + n_chars] |
| if r.random() < 0.25: |
| state = {"source": "encyclopedia", "lang": lang, "text": passage} |
| else: |
| state = passage |
| qs = {} |
| for i, qi in enumerate(r.sample(range(len(QUESTIONS)), r.randint(1, 4))): |
| qs[f"q{i}"] = QUESTIONS[qi](r, lang) |
| reqs.append((state, qs)) |
| r.shuffle(reqs) |
| return reqs |
|
|
