#!/usr/bin/env python3 """Generate short (3-6 word) standalone semantic examples for targeted patterns.""" import json import requests import os import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from config import OLLAMA_URL, OLLAMA_MODEL RAW_DIR = "data/raw" def generate(lang_code, lang_name, topic, patterns, num_rounds=50): filepath = os.path.join(RAW_DIR, f"{lang_code}_short_semantic.jsonl") count = 0 for i in range(num_rounds): prompt = ( f"Generate 5 very short {lang_name} SEMANTIC queries (3-6 words only). " f"Topic: {topic}\n\n" f"Required patterns:\n{patterns}\n\n" f"OUTPUT RULES:\n" f"- Each query MUST be 3-6 words only\n" f"- No greetings, no commands, no chit-chat\n" f"- Self-contained semantic content (personal fact or preference)\n" f"- Use different names/items/professions each time\n" f"- Vary the sentence structure\n" f"- Output ONLY 5 lines of valid JSONL, nothing else\n" f'Format: {{"text": "", "language": "{lang_code}", "label": "SEMANTIC"}}' ) try: resp = requests.post(OLLAMA_URL, json={ "model": OLLAMA_MODEL, "messages": [{"role": "user", "content": prompt}], "stream": False, "options": {"temperature": 0.9, "num_predict": 1024} }, timeout=60) content = resp.json()["message"]["content"] added = 0 with open(filepath, "a") as f: for line in content.strip().split("\n"): line = line.strip() if not line or line.startswith("```"): continue try: obj = json.loads(line) text = obj.get("text", "") word_count = len(text.split()) if (obj.get("label") == "SEMANTIC" and obj.get("language") == lang_code and 3 <= word_count <= 7 and len(text) < 80): f.write(json.dumps(obj, ensure_ascii=False) + "\n") added += 1 except: pass count += added except Exception as e: pass sys.stdout.write(f"\r {lang_name}: round {i+1}/{num_rounds}, {count} total ") sys.stdout.flush() print(f"\n Done: {count} examples -> {filepath}") return count if __name__ == "__main__": os.makedirs(RAW_DIR, exist_ok=True) print("Generating short Hindi SEMANTIC queries...") generate("hi", "Hindi", "Personal identity, preferences, and relationships", '- "mera naam X hai" (Amit, Priya, Vikram, Sunita, Arjun, Kavita, etc.)\n' '- "mujhe X pasand/nahi pasand hai" (food, activities, etc.)\n' '- "main X hoon" (doctor, teacher, engineer, artist, student, lawyer)\n' '- "meri X Y hai" (family, possessions)\n' '- "mera X Y hai" (possessions, attributes)\n' '- "mujhe X se allergy hai"\n' '- "meri umar X hai"\n' 'Output 3-6 word Hindi Devanagari sentences ONLY.', num_rounds=80) print("\nGenerating short English SEMANTIC queries...") generate("en", "English", "Personal identity, preferences, and relationships", '- "my name is X"\n' '- "I am a X" (doctor, teacher, engineer, artist, etc.)\n' '- "I love/like/hate/enjoy X"\n' '- "my favorite X is Y"\n' '- "I prefer X over Y"\n' '- "my X is a Y" (family relationships)\n' '- "I work as a X"\n' '- "I live in X"\n' 'Output 3-6 word English sentences ONLY.', num_rounds=80) print("\nDone!")