#!/usr/bin/env python3 """Generate extra short semantic examples for targeted patterns.""" import json import requests import os import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from config import OLLAMA_URL, OLLAMA_MODEL, RAW_DIR def generate_batch(lang_code, lang_name, patterns, filename): filepath = os.path.join(RAW_DIR, filename) total = 0 for round_num in range(10): prompt = ( f"Generate 20 very short {lang_name} SEMANTIC queries. " f"These are personal facts, preferences, or identity statements.\n\n" f"{patterns}\n\n" f"Each query must be 3-6 words only, self-contained semantic content.\n" f"NO greetings, NO commands, NO chit-chat.\n" f"Use different names, items, professions each time.\n\n" f"Output JSONL only:\n" f'{{"text": "", "language": "{lang_code}", "label": "SEMANTIC"}}' ) try: resp = requests.post(OLLAMA_URL, json={ "model": OLLAMA_MODEL, "messages": [{"role": "user", "content": prompt}], "stream": False, "options": {"temperature": 0.8, "num_predict": 2048} }, timeout=60) content = resp.json()["message"]["content"] batch_count = 0 with open(filepath, "a") as f: for line in content.strip().split("\n"): line = line.strip() if not line or line.startswith("```"): continue try: obj = json.loads(line) text = obj.get("text", "") if (obj.get("label") == "SEMANTIC" and obj.get("language") == lang_code and 10 < len(text) < 80 and not any(c in text for c in "{}[]()")): f.write(json.dumps(obj, ensure_ascii=False) + "\n") batch_count += 1 except json.JSONDecodeError: pass total += batch_count print(f" Round {round_num+1}/10: {batch_count} examples (total: {total})") if batch_count == 0: print(" No valid examples generated, stopping early") break except Exception as e: print(f" Error: {e}") continue return total if __name__ == "__main__": print("Generating extra short Hindi SEMANTIC examples...") hi_total = generate_batch( "hi", "Hindi", "Simple personal statements in Devanagari Hindi:\n" '- "mera naam X hai" with different Indian names (Sunita, Amit, Priya, Vikram, Kavita, etc.)\n' '- "mujhe X pasand hai" with various foods, activities, colors, books\n' '- "main X hoon" with professions (teacher, student, doctor, engineer, artist, lawyer)\n' '- "meri X Y hai" with family and possessions\n' 'EXAMPLE: {"text": "मेरा नाम अमित है", "language": "hi", "label": "SEMANTIC"}', "hi_semantic_extra.jsonl" ) print(f"\nGenerating extra short English SEMANTIC examples...") en_total = generate_batch( "en", "English", "Simple personal preference and fact statements:\n" '- "I love/like/enjoy X" with various foods, activities, hobbies\n' '- "my name is X" with different names\n' '- "I am a X" with professions, roles\n' '- "my favorite X is Y" with various categories\n' '- "I prefer X" or "I hate X" with various items\n' 'EXAMPLE: {"text": "I love spicy food", "language": "en", "label": "SEMANTIC"}', "en_semantic_extra.jsonl" ) print(f"\nDone!") print(f" Hindi extra: check data/raw/hi_semantic_extra.jsonl") print(f" English extra: check data/raw/en_semantic_extra.jsonl")