from openai import OpenAI import json, os source_language = "Bengali" if source_language == "English": source_lang_code = "en" elif source_language == "Spanish": source_lang_code = "es" elif source_language == "French": source_lang_code = "fr" elif source_language == "Portuguese": source_lang_code = "pt" elif source_language == "Bengali": source_lang_code = "bn" else: assert False, "Unsupported language" print(f"{source_language}") with open( "/home/mshahidul/readctrl/prompts/syn_data_generation/syn_data_gen_diff_label_Bangla.txt", "r", encoding="utf-8", ) as f: prompt_template = f.read() api_file = "/home/mshahidul/api_new.json" with open(api_file, "r", encoding="utf-8") as f: api_keys = json.load(f) openai_api_key = api_keys["openai"] client = OpenAI(api_key=openai_api_key) def openai_return(prompt, model="gpt-5"): """Send a prompt to GPT and parse JSON.""" response = client.chat.completions.create( model=model, messages=[ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": prompt} ] ) content = response.choices[0].message.content.strip() cleaned = content.replace("```json", "").replace("```", "").strip() try: return json.loads(cleaned) except json.JSONDecodeError: print("⚠️ JSON parse failed — storing raw text.") return cleaned input_path = "/home/mshahidul/readctrl/data/translated_data/translation_wo_judge/multiclinsum_gs_train_en2bn_gemma(0_200).json" save_path = f"/home/mshahidul/readctrl/data/synthetic_dataset_diff_labels/syn_data_diff_labels_{source_lang_code}_0_80.json" res=[] if os.path.exists(save_path): with open(save_path, "r", encoding="utf-8") as f: res = json.load(f) import tqdm with open(input_path, "r", encoding="utf-8") as f: data = json.load(f) for idx, item in tqdm.tqdm(enumerate(data)): fulltext_bn = item.get("translated_fulltext") summary_bn = item.get("translated_summary") if not fulltext_bn or not summary_bn: print(f"Skipping idx={idx}, id={item.get('id')} due to missing translated fields.") continue prompt = ( prompt_template .replace("<<>>", fulltext_bn) .replace("<<>>", source_language) .replace("<<>>", summary_bn) ) # import ipdb; ipdb.set_trace() sample = openai_return(prompt, model="gpt-5") res.append({ "id": item["id"], "index": idx , "fulltext": fulltext_bn, "diff_label_texts": sample }) if len(res) % 2 == 0: with open(save_path, "w", encoding="utf-8") as f: json.dump(res, f, indent=2, ensure_ascii=False) print(f"Saved {len(res)} samples so far.") with open(save_path, "w", encoding="utf-8") as f: json.dump(res, f, indent=2, ensure_ascii=False)