| |
| """Generate extra short semantic examples for targeted patterns.""" |
| import json |
| import requests |
| import os |
| import sys |
|
|
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| from config import OLLAMA_URL, OLLAMA_MODEL, RAW_DIR |
|
|
| def generate_batch(lang_code, lang_name, patterns, filename): |
| filepath = os.path.join(RAW_DIR, filename) |
| total = 0 |
| |
| for round_num in range(10): |
| prompt = ( |
| f"Generate 20 very short {lang_name} SEMANTIC queries. " |
| f"These are personal facts, preferences, or identity statements.\n\n" |
| f"{patterns}\n\n" |
| f"Each query must be 3-6 words only, self-contained semantic content.\n" |
| f"NO greetings, NO commands, NO chit-chat.\n" |
| f"Use different names, items, professions each time.\n\n" |
| f"Output JSONL only:\n" |
| f'{{"text": "<query>", "language": "{lang_code}", "label": "SEMANTIC"}}' |
| ) |
| |
| try: |
| resp = requests.post(OLLAMA_URL, json={ |
| "model": OLLAMA_MODEL, |
| "messages": [{"role": "user", "content": prompt}], |
| "stream": False, |
| "options": {"temperature": 0.8, "num_predict": 2048} |
| }, timeout=60) |
| |
| content = resp.json()["message"]["content"] |
| batch_count = 0 |
| |
| with open(filepath, "a") as f: |
| for line in content.strip().split("\n"): |
| line = line.strip() |
| if not line or line.startswith("```"): |
| continue |
| try: |
| obj = json.loads(line) |
| text = obj.get("text", "") |
| if (obj.get("label") == "SEMANTIC" and |
| obj.get("language") == lang_code and |
| 10 < len(text) < 80 and |
| not any(c in text for c in "{}[]()")): |
| f.write(json.dumps(obj, ensure_ascii=False) + "\n") |
| batch_count += 1 |
| except json.JSONDecodeError: |
| pass |
| |
| total += batch_count |
| print(f" Round {round_num+1}/10: {batch_count} examples (total: {total})") |
| |
| if batch_count == 0: |
| print(" No valid examples generated, stopping early") |
| break |
| |
| except Exception as e: |
| print(f" Error: {e}") |
| continue |
| |
| return total |
|
|
|
|
| if __name__ == "__main__": |
| print("Generating extra short Hindi SEMANTIC examples...") |
| hi_total = generate_batch( |
| "hi", "Hindi", |
| "Simple personal statements in Devanagari Hindi:\n" |
| '- "mera naam X hai" with different Indian names (Sunita, Amit, Priya, Vikram, Kavita, etc.)\n' |
| '- "mujhe X pasand hai" with various foods, activities, colors, books\n' |
| '- "main X hoon" with professions (teacher, student, doctor, engineer, artist, lawyer)\n' |
| '- "meri X Y hai" with family and possessions\n' |
| 'EXAMPLE: {"text": "मेरा नाम अमित है", "language": "hi", "label": "SEMANTIC"}', |
| "hi_semantic_extra.jsonl" |
| ) |
| |
| print(f"\nGenerating extra short English SEMANTIC examples...") |
| en_total = generate_batch( |
| "en", "English", |
| "Simple personal preference and fact statements:\n" |
| '- "I love/like/enjoy X" with various foods, activities, hobbies\n' |
| '- "my name is X" with different names\n' |
| '- "I am a X" with professions, roles\n' |
| '- "my favorite X is Y" with various categories\n' |
| '- "I prefer X" or "I hate X" with various items\n' |
| 'EXAMPLE: {"text": "I love spicy food", "language": "en", "label": "SEMANTIC"}', |
| "en_semantic_extra.jsonl" |
| ) |
| |
| print(f"\nDone!") |
| print(f" Hindi extra: check data/raw/hi_semantic_extra.jsonl") |
| print(f" English extra: check data/raw/en_semantic_extra.jsonl") |
|
|
