File size: 4,172 Bytes
2d2f883 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """Calibration requests in Prism's own input format, built from multilingual Wikipedia passages (11 languages).
Disjoint from the evaluation suite (different texts and question wording)."""
import json
import random
LANG_NAMES = {"en": "English", "hi": "Hindi", "bn": "Bengali", "ta": "Tamil", "te": "Telugu", "mr": "Marathi",
"gu": "Gujarati", "kn": "Kannada", "ml": "Malayalam", "or": "Odia", "pa": "Punjabi"}
QUESTIONS = [
lambda r, l: {"type": "choice", "instructions": "What is the main subject area of this text?",
"criteria": {"history": "history and past events", "science": "science and technology",
"geography": "places and geography", "people": "a biography of a person",
"culture": "arts, religion, culture", "politics": "government and politics",
"sports": None, "other": None}},
lambda r, l: {"type": "choice", "instructions": "In which language is the passage written?",
"criteria": dict(r.sample(sorted(LANG_NAMES.items()), 4) + [(l, LANG_NAMES[l])])},
lambda r, l: {"type": "yes_no", "instructions": "Does the passage mention a specific year or date?"},
lambda r, l: {"type": "yes_no", "instructions": "Is this passage about a living person?"},
lambda r, l: {"type": "yes_no", "instructions": "Would this text be suitable for a children's encyclopedia?"},
lambda r, l: {"type": "yes_no", "instructions": "Does the text contain a factual error?"},
lambda r, l: {"type": "score", "instructions": "How technical is the language of this passage?",
"criteria": ["very simple", "simple", "moderate", "technical", "highly specialised"]},
lambda r, l: {"type": "score", "instructions": "How well written is this text?",
"criteria": {"poor": "many errors", "fair": "some problems", "good": "clear", "great": "excellent prose"}},
lambda r, l: {"type": "score", "instructions": "How likely is a general reader to find this interesting?",
"criteria": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]},
lambda r, l: {"type": "choice", "instructions": "Which audience is this text written for?",
"criteria": {"general": "general public", "student": "students", "expert": "domain experts"}},
lambda r, l: {"type": "choice", "instructions": "What tone does the passage have?",
"criteria": {"neutral": None, "positive": None, "negative": None, "promotional": "advertising"}},
lambda r, l: {"type": "yes_no", "instructions": "किया यह पाठ किसी स्थान के बारे में है?" if l != "en" else
"Is this text about a place?"},
lambda r, l: {"type": "choice", "instructions": "If a user submitted this as a support ticket, which queue fits best?",
"criteria": {"billing": "payments and refunds", "tech": "technical issue", "spam": "not a real ticket",
"general": "general enquiry"}},
lambda r, l: {"type": "score", "instructions": "How urgent would it be to act on this information?",
"criteria": ["not urgent", "low", "medium", "high", "critical"]},
]
def make_requests(wiki_path, seed=0, n_per_lang=400):
wiki = json.load(open(wiki_path))
r = random.Random(seed)
reqs = []
for lang, texts in wiki.items():
for t in texts[:n_per_lang]:
u = r.random()
n_chars = r.randint(150, 900) if u < 0.55 else (r.randint(900, 3500) if u < 0.95 else r.randint(3500, 12000))
start = r.randint(0, max(0, len(t) - n_chars)) if len(t) > n_chars else 0
passage = t[start:start + n_chars]
if r.random() < 0.25:
state = {"source": "encyclopedia", "lang": lang, "text": passage}
else:
state = passage
qs = {}
for i, qi in enumerate(r.sample(range(len(QUESTIONS)), r.randint(1, 4))):
qs[f"q{i}"] = QUESTIONS[qi](r, lang)
reqs.append((state, qs))
r.shuffle(reqs)
return reqs
|