| from openai import OpenAI |
| import json, os |
|
|
| source_language = "Bengali" |
| if source_language == "English": |
| source_lang_code = "en" |
| elif source_language == "Spanish": |
| source_lang_code = "es" |
| elif source_language == "French": |
| source_lang_code = "fr" |
| elif source_language == "Portuguese": |
| source_lang_code = "pt" |
| elif source_language == "Bengali": |
| source_lang_code = "bn" |
| else: |
| assert False, "Unsupported language" |
| print(f"{source_language}") |
| with open( |
| "/home/mshahidul/readctrl/prompts/syn_data_generation/syn_data_gen_diff_label_Bangla.txt", |
| "r", |
| encoding="utf-8", |
| ) as f: |
| prompt_template = f.read() |
|
|
|
|
| api_file = "/home/mshahidul/api_new.json" |
| with open(api_file, "r", encoding="utf-8") as f: |
| api_keys = json.load(f) |
| openai_api_key = api_keys["openai"] |
|
|
| client = OpenAI(api_key=openai_api_key) |
|
|
|
|
| def openai_return(prompt, model="gpt-5"): |
| """Send a prompt to GPT and parse JSON.""" |
| response = client.chat.completions.create( |
| model=model, |
| messages=[ |
| {"role": "system", "content": "You are a helpful assistant."}, |
| {"role": "user", "content": prompt} |
| ] |
| ) |
| content = response.choices[0].message.content.strip() |
| cleaned = content.replace("```json", "").replace("```", "").strip() |
| try: |
| return json.loads(cleaned) |
| except json.JSONDecodeError: |
| print("⚠️ JSON parse failed — storing raw text.") |
| return cleaned |
|
|
| input_path = "/home/mshahidul/readctrl/data/translated_data/translation_wo_judge/multiclinsum_gs_train_en2bn_gemma(0_200).json" |
| save_path = f"/home/mshahidul/readctrl/data/synthetic_dataset_diff_labels/syn_data_diff_labels_{source_lang_code}_0_80.json" |
| res=[] |
| if os.path.exists(save_path): |
| with open(save_path, "r", encoding="utf-8") as f: |
| res = json.load(f) |
| import tqdm |
| with open(input_path, "r", encoding="utf-8") as f: |
| data = json.load(f) |
| for idx, item in tqdm.tqdm(enumerate(data)): |
| fulltext_bn = item.get("translated_fulltext") |
| summary_bn = item.get("translated_summary") |
| if not fulltext_bn or not summary_bn: |
| print(f"Skipping idx={idx}, id={item.get('id')} due to missing translated fields.") |
| continue |
|
|
| prompt = ( |
| prompt_template |
| .replace("<<<FULL_TEXT>>>", fulltext_bn) |
| .replace("<<<SOURCE_LANGUAGE>>>", source_language) |
| .replace("<<<GOLD_SUMMARY>>>", summary_bn) |
| ) |
| |
| sample = openai_return(prompt, model="gpt-5") |
|
|
| res.append({ |
| "id": item["id"], |
| "index": idx , |
| "fulltext": fulltext_bn, |
| "diff_label_texts": sample |
| }) |
|
|
| if len(res) % 2 == 0: |
| with open(save_path, "w", encoding="utf-8") as f: |
| json.dump(res, f, indent=2, ensure_ascii=False) |
| print(f"Saved {len(res)} samples so far.") |
|
|
| with open(save_path, "w", encoding="utf-8") as f: |
| json.dump(res, f, indent=2, ensure_ascii=False) |
|
|
|
|
