| """New schema families with shared identifier prefixes and explicit relation contrasts.""" |
| import argparse |
| import json |
| from pathlib import Path |
| import random |
| import re |
| import shutil |
| import string |
| from tokenizers import Tokenizer |
| from sqlglot import Tokenizer as SQLTokenizer |
| from tinyquery.data import build_record,load_templates,serialize,DOMAINS |
| from tinyquery.recipes import RECIPES |
|
|
| RELATIONS={ |
| 'en':{'gt':['strictly above','greater than','higher than','over'], |
| 'lt':['strictly below','less than','lower than','under'], |
| 'gte':['at least','not less than','no lower than','not below','greater than or equal to'], |
| 'lte':['at most','not greater than','no higher than','not above','less than or equal to']}, |
| 'hi':{'gt':['{number} से ज़्यादा','{number} से अधिक'], |
| 'lt':['{number} से कम','{number} से नीचे'], |
| 'gte':['{number} से कम नहीं','{number} या उससे अधिक','कम से कम {number}'], |
| 'lte':['{number} से अधिक नहीं','{number} या उससे कम','अधिकतम {number}']}, |
| 'hinglish':{'gt':['{number} se zyada','{number} se upar'], |
| 'lt':['{number} se kam','{number} se neeche'], |
| 'gte':['{number} se kam nahi','{number} ya usse zyada','kam se kam {number}'], |
| 'lte':['{number} se zyada nahi','{number} ya usse kam','zyada se zyada {number}']}} |
| WRITES={ |
| 'en':['Empty {table} completely.','Purge every entry stored in {table}.','Erase the entire contents of {table}.','Blow away the stored rows in {table}.','Wipe the data inside {table}.'], |
| 'noisy_en':['{table} all data remove pls','empty {table} whole table now','wipe every {table} row please','purge {table} data all'], |
| 'hi':['{table} की सभी पंक्तियाँ मिटा दो।','{table} में मौजूद पूरा डेटा हटा दो।','{table} को पूरी तरह खाली कर दो।'], |
| 'hinglish':['{table} ka saara data hata do.','{table} ko bilkul khaali kar do.','{table} ki har row mita do.']} |
|
|
|
|
| def contrast(row,rng): |
| op=row['operation'];lang=row['language'];s=row['slots'] |
| if op=='write':return rng.choice(WRITES[lang]).format(**s) |
| if lang in ['en','noisy_en']: |
| relation=rng.choice(RELATIONS['en'][op]) |
| template=('For {table}, return records whose {numeric} is '+relation+' {number}.') if lang=='en' else ('need {table} rows, {numeric} '+relation+' {number} pls') |
| elif lang=='hi': |
| template='{table} से केवल वे रिकॉर्ड चाहिए जिनमें {numeric} '+rng.choice(RELATIONS[lang][op])+' हो।' |
| else: |
| template='{table} mein sirf wahi rows chahiye jinka {numeric} '+rng.choice(RELATIONS[lang][op])+' ho.' |
| return template.format(**s) |
|
|
|
|
| def main(): |
| p=argparse.ArgumentParser();p.add_argument('--source',default='data/tinyquery-v5');p.add_argument('--out',default='data/tinyquery-v6') |
| p.add_argument('--templates',default='data/tinyquery/templates.jsonl');p.add_argument('--scenarios',type=int,default=6000) |
| args=p.parse_args();source=Path(args.source);out=Path(args.out);out.mkdir(parents=True,exist_ok=True) |
| for name in ['validation.jsonl','test.jsonl','manual.jsonl','tokenizer.json']:shutil.copy2(source/name,out/name) |
| tokenizer=Tokenizer.from_file(str(source/'tokenizer.json')) |
| chunks=sorted({tokenizer.decode([i]) for i in range(tokenizer.get_vocab_size()) if re.fullmatch('[a-z]{1,8}',tokenizer.decode([i]))}) |
| forbidden=set(sum(DOMAINS.values(),[]));templates=load_templates(args.templates);rng=random.Random(99606) |
| ops=list(RECIPES)+['join','join_filter']*5+['gt','lt','gte','lte','write']*3+['list_tables','describe','missing_schema','sql_error']*2 |
| count=0;contrasts=0;domains=set();heldout_phrases={} |
| for op,languages in templates.items(): |
| for lang,phrases in languages.items():heldout_phrases[(op,lang)]=phrases[-2:] |
| with (out/'train.jsonl').open('w') as stream: |
| with (source/'train.jsonl').open() as previous:shutil.copyfileobj(previous,stream) |
| for i in range(args.scenarios): |
| while True: |
| domain=''.join(rng.choices(chunks,k=rng.choice([3,4,5]))) if rng.random()<.8 else ''.join(rng.choices(string.ascii_lowercase,k=rng.randrange(6,15))) |
| if 6<=len(domain)<=24 and domain not in forbidden and domain not in domains and domain.upper() not in SQLTokenizer.KEYWORDS:break |
| domains.add(domain);op=rng.choice(ops);backend='supabase' if rng.random()<.65 else 'mysql' |
| for row in build_record(op,domain,i,backend,'train',templates,rng): |
| row['id']+='_curriculum_99606';row['scenario_id']='curriculum_'+row['scenario_id'];row['sample_weight']=6 |
| row['provenance']+='; new schema family with related table/parent/project identifiers' |
| if op in ['gt','lt','gte','lte','write'] and rng.random()<.75: |
| question=contrast(row,rng) |
| |
| heldout={text.format(**row['slots']).casefold().strip() for text in heldout_phrases[(op,row['language'])]} |
| if question.casefold().strip() not in heldout: |
| row['question']=question;row['prompt']=serialize(row['context'],question) |
| row['template_index']=-1;row['provenance']+='; programmatically authored contrastive relation/write phrasing';contrasts+=1 |
| stream.write(json.dumps(row,ensure_ascii=False)+'\n');count+=1 |
| report={'new_scenario_families':len(domains),'new_examples':count,'contrastive_examples':contrasts,'sample_weight':6} |
| (out/'grounding-stats.json').write_text(json.dumps(report,indent=2));print(report) |
|
|
|
|
| if __name__=='__main__':main() |
|
|
