Spaces:
Running
Running
File size: 4,880 Bytes
160aacb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | import os
import csv
import json
import asyncio
from openai import AsyncOpenAI
from dotenv import load_dotenv
load_dotenv()
# Configure Teacher Model
AI_API_KEY = os.getenv("QWEN_API_KEY", "your-api-key-here")
AI_BASE_URL = os.getenv("QWEN_BASE_URL", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1")
TEACHER_MODEL_NAME = os.getenv("TEACHER_MODEL_NAME", "qwen3-coder-480b-a35b")
client = AsyncOpenAI(
api_key=AI_API_KEY,
base_url=AI_BASE_URL
)
IEDID_DIR = "IEDID"
OUTPUT_FILE = "synthetic_slm_dataset.jsonl"
async def generate_synthetic_data(dialect, utterance, clarification):
system_prompt = (
f"You are a master linguist specializing in {dialect} and data augmentation for SLM fine-tuning.\n"
f"I will provide a human-verified conversation pair.\n"
f"Dialect Utterance: '{utterance}'\n"
f"Standard English Meaning: '{clarification}'\n\n"
f"Generate 3 highly accurate, synthetically varied conversation pairs that capture the exact same "
f"linguistic rules, phonetic syntax, and cultural tone as the example.\n"
f"Respond ONLY in valid JSON array format, where each object has 'prompt' (Standard English) and 'completion' ({dialect})."
)
try:
response = await client.chat.completions.create(
model=TEACHER_MODEL_NAME,
messages=[
{"role": "system", "content": system_prompt},
{"role": "user", "content": "Generate 3 synthetic pairs in a JSON array."}
],
temperature=0.7,
max_tokens=1024
)
content = response.choices[0].message.content.strip()
# Fallback parsing if wrapped in markdown
if content.startswith("```json"):
content = content[7:-3]
elif content.startswith("```"):
content = content[3:-3]
data = json.loads(content)
# Expected format: [{"prompt": "...", "completion": "..."}, ...]
# If wrapped in a key
if isinstance(data, dict):
for key in data:
if isinstance(data[key], list):
return data[key]
return []
return data if isinstance(data, list) else []
except Exception as e:
print(f"Error generating data for '{utterance}': {e}")
return []
async def process_csv(filepath):
print(f"Processing {filepath}...")
dataset = []
with open(filepath, 'r', encoding='utf-8') as f:
reader = csv.DictReader(f)
rows = list(reader)[:10] # Limit to first 10 per file for demonstration/testing speed
for row in rows:
utterance = row.get('Utterance')
clarification = row.get('Clarification')
dialect = row.get('Dialect', 'Unknown Dialect')
if not utterance or not clarification:
continue
print(f" Distilling variations for: {utterance}")
synthetic_pairs = await generate_synthetic_data(dialect, utterance, clarification)
for pair in synthetic_pairs:
if "prompt" in pair and "completion" in pair:
dataset.append({
"system": f"You are an expert polyglot interpreter. Translate Standard English to {dialect}.",
"user": pair["prompt"],
"assistant": pair["completion"]
})
return dataset
PRIORITY_DIALECTS = [
"African American Vernacular English",
"Nigerian Pidgin English",
"Gyeongsang Satoori",
"Jeolla Satoori",
"Chungcheong Satoori",
"Jeju Satoori",
"Gangwon Satoori",
"Hamgyong Satoori"
]
async def main():
print(f"Starting Knowledge Distillation Pipeline (Teacher Model: {TEACHER_MODEL_NAME})...")
all_synthetic_data = []
# Read IEDID CSVs
if os.path.exists(IEDID_DIR):
for filename in os.listdir(IEDID_DIR):
# Only process if it's a CSV, not minted, and matches a priority dialect
is_priority = any(dialect in filename for dialect in PRIORITY_DIALECTS)
if filename.endswith(".csv") and "minted" not in filename and is_priority:
filepath = os.path.join(IEDID_DIR, filename)
data = await process_csv(filepath)
all_synthetic_data.extend(data)
# Save to JSONL
with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
for entry in all_synthetic_data:
f.write(json.dumps(entry) + '\n')
print(f"\nSuccessfully generated {len(all_synthetic_data)} synthetic training pairs!")
print(f"Saved to {OUTPUT_FILE}. Ready for HuggingFace AutoTrain or LoRA fine-tuning.")
if __name__ == "__main__":
asyncio.run(main())
|