| |
| """Generate short (3-6 word) standalone semantic examples for targeted patterns.""" |
| import json |
| import requests |
| import os |
| import sys |
|
|
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| from config import OLLAMA_URL, OLLAMA_MODEL |
|
|
| RAW_DIR = "data/raw" |
|
|
| def generate(lang_code, lang_name, topic, patterns, num_rounds=50): |
| filepath = os.path.join(RAW_DIR, f"{lang_code}_short_semantic.jsonl") |
| count = 0 |
| |
| for i in range(num_rounds): |
| prompt = ( |
| f"Generate 5 very short {lang_name} SEMANTIC queries (3-6 words only). " |
| f"Topic: {topic}\n\n" |
| f"Required patterns:\n{patterns}\n\n" |
| f"OUTPUT RULES:\n" |
| f"- Each query MUST be 3-6 words only\n" |
| f"- No greetings, no commands, no chit-chat\n" |
| f"- Self-contained semantic content (personal fact or preference)\n" |
| f"- Use different names/items/professions each time\n" |
| f"- Vary the sentence structure\n" |
| f"- Output ONLY 5 lines of valid JSONL, nothing else\n" |
| f'Format: {{"text": "<query>", "language": "{lang_code}", "label": "SEMANTIC"}}' |
| ) |
| |
| try: |
| resp = requests.post(OLLAMA_URL, json={ |
| "model": OLLAMA_MODEL, |
| "messages": [{"role": "user", "content": prompt}], |
| "stream": False, |
| "options": {"temperature": 0.9, "num_predict": 1024} |
| }, timeout=60) |
| |
| content = resp.json()["message"]["content"] |
| added = 0 |
| |
| with open(filepath, "a") as f: |
| for line in content.strip().split("\n"): |
| line = line.strip() |
| if not line or line.startswith("```"): |
| continue |
| try: |
| obj = json.loads(line) |
| text = obj.get("text", "") |
| word_count = len(text.split()) |
| if (obj.get("label") == "SEMANTIC" |
| and obj.get("language") == lang_code |
| and 3 <= word_count <= 7 |
| and len(text) < 80): |
| f.write(json.dumps(obj, ensure_ascii=False) + "\n") |
| added += 1 |
| except: |
| pass |
| |
| count += added |
| |
| except Exception as e: |
| pass |
| |
| sys.stdout.write(f"\r {lang_name}: round {i+1}/{num_rounds}, {count} total ") |
| sys.stdout.flush() |
| |
| print(f"\n Done: {count} examples -> {filepath}") |
| return count |
|
|
| if __name__ == "__main__": |
| os.makedirs(RAW_DIR, exist_ok=True) |
| |
| print("Generating short Hindi SEMANTIC queries...") |
| generate("hi", "Hindi", |
| "Personal identity, preferences, and relationships", |
| '- "mera naam X hai" (Amit, Priya, Vikram, Sunita, Arjun, Kavita, etc.)\n' |
| '- "mujhe X pasand/nahi pasand hai" (food, activities, etc.)\n' |
| '- "main X hoon" (doctor, teacher, engineer, artist, student, lawyer)\n' |
| '- "meri X Y hai" (family, possessions)\n' |
| '- "mera X Y hai" (possessions, attributes)\n' |
| '- "mujhe X se allergy hai"\n' |
| '- "meri umar X hai"\n' |
| 'Output 3-6 word Hindi Devanagari sentences ONLY.', |
| num_rounds=80) |
| |
| print("\nGenerating short English SEMANTIC queries...") |
| generate("en", "English", |
| "Personal identity, preferences, and relationships", |
| '- "my name is X"\n' |
| '- "I am a X" (doctor, teacher, engineer, artist, etc.)\n' |
| '- "I love/like/hate/enjoy X"\n' |
| '- "my favorite X is Y"\n' |
| '- "I prefer X over Y"\n' |
| '- "my X is a Y" (family relationships)\n' |
| '- "I work as a X"\n' |
| '- "I live in X"\n' |
| 'Output 3-6 word English sentences ONLY.', |
| num_rounds=80) |
|
|
| print("\nDone!") |
|
|
