Spaces:
Sleeping
Sleeping
| # import gradio as gr | |
| # from transformers import BlipProcessor, BlipForConditionalGeneration | |
| # from gtts import gTTS | |
| # import io | |
| # from PIL import Image | |
| # # ------------------------------- | |
| # # Load BLIP-base model (lighter version) | |
| # # ------------------------------- | |
| # processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large") | |
| # model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large") | |
| # # ------------------------------- | |
| # # Generate caption function | |
| # # ------------------------------- | |
| # # def generate_caption_tts(image): | |
| # # caption = generate_caption(model, processor, image) | |
| # # audio_file = text_to_audio_file(caption) | |
| # # return caption, audio_file # return file path, not BytesIO | |
| # # ------------------------------- | |
| # # Convert text to speech using gTTS | |
| # # ------------------------------- | |
| # import tempfile | |
| # import pyttsx3 | |
| # def text_to_audio_file(text): | |
| # # Create a temporary file | |
| # tmp_file = tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) | |
| # tmp_path = tmp_file.name | |
| # tmp_file.close() | |
| # engine = pyttsx3.init() | |
| # engine.save_to_file(text, tmp_path) | |
| # engine.runAndWait() | |
| # return tmp_path | |
| # def generate_caption_from_image(model, processor, image): | |
| # # image: PIL.Image | |
| # inputs = processor(images=image, return_tensors="pt") | |
| # out = model.generate(**inputs) | |
| # caption = processor.decode(out[0], skip_special_tokens=True) | |
| # return caption | |
| # # ------------------------------- | |
| # # Gradio interface: Caption + Audio | |
| # # ------------------------------- | |
| # def generate_caption_tts(image): | |
| # caption = generate_caption_from_image(model, processor, image) # uses global model/processor | |
| # # audio_file = text_to_audio_file(caption) | |
| # return caption | |
| # interface = gr.Interface( | |
| # fn=generate_caption_tts, | |
| # inputs=gr.Image(type="numpy"), | |
| # outputs=[gr.Textbox(label="Generated Caption")], | |
| # title="Image Captioning for Visually Impaired", | |
| # description="Upload an image, get a caption and audio description." | |
| # ) | |
| # interface.launch() | |
| # # demo.launch(share=True) | |
| import gradio as gr | |
| from transformers import ( | |
| BlipProcessor, | |
| BlipForConditionalGeneration, | |
| BlipForQuestionAnswering, | |
| pipeline | |
| ) | |
| moderation_model = pipeline( | |
| "text-classification", | |
| model="Vrandan/Comment-Moderation", | |
| return_all_scores=True | |
| ) | |
| from PIL import Image | |
| import torch | |
| from gtts import gTTS | |
| import tempfile | |
| # ---------------------- | |
| # Device setup | |
| # ---------------------- | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| # ---------------------- | |
| # Load Models Once | |
| # ---------------------- | |
| print("🔄 Loading models...") | |
| # Captioning | |
| caption_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large") | |
| caption_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large").to(device) | |
| # VQA | |
| vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") | |
| vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base").to(device) | |
| # Translation | |
| translation_models = { | |
| "Hindi": pipeline("translation", model="Helsinki-NLP/opus-mt-en-hi"), | |
| "French": pipeline("translation", model="Helsinki-NLP/opus-mt-en-fr"), | |
| "Spanish": pipeline("translation", model="Helsinki-NLP/opus-mt-en-es"), | |
| } | |
| # Safety Moderation Pipeline | |
| moderation_model = pipeline("text-classification", model="unitary/toxic-bert") | |
| print("✅ All models loaded!") | |
| # ---------------------- | |
| # Safety Filter Function | |
| # ---------------------- | |
| def is_caption_safe(caption): | |
| try: | |
| votes = moderation_model(caption) | |
| # If return_all_scores=True, it's [[{label, score}, ...]] | |
| if isinstance(votes, list) and isinstance(votes[0], list): | |
| votes = votes[0] | |
| # Now safe to loop | |
| for item in votes: | |
| if isinstance(item, dict) and item.get("label") in ["V", "V2"] and item.get("score", 0) > 0.5: | |
| return False | |
| except Exception as e: | |
| print("⚠️ Moderation failed:", e) | |
| # Fallback keywords | |
| unsafe_keywords = [ | |
| "gun", "blood", "skull", "kill", "corpse", "gore", "knife", "weapon", | |
| "fire", "murder", "dead", "death", "suicide", "bomb", "explosion", | |
| "terrorist", "assault", "stab", "shoot", "pistol", "rifle", "shotgun", | |
| "grenade", "horror", "beheaded", "torture", "hostage", "rape", | |
| "war", "massacre", "chainsaw", "poison", "strangle", "hang", "drown" | |
| ] | |
| if any(word in caption.lower() for word in unsafe_keywords): | |
| return False | |
| return True | |
| # ---------------------- | |
| # Caption + Translate + Speak | |
| # ---------------------- | |
| def generate_caption_translate_speak(image, target_lang): | |
| # Step 1: Caption | |
| inputs = caption_processor(images=image, return_tensors="pt").to(device) | |
| with torch.no_grad(): | |
| out = caption_model.generate(**inputs, max_new_tokens=50) | |
| english_caption = caption_processor.decode(out[0], skip_special_tokens=True) | |
| # Step 1.5: Safety Check | |
| if not is_caption_safe(english_caption): | |
| return "⚠️ Warning: Unsafe or inappropriate content detected!", "", None | |
| # Step 2: Translate | |
| if target_lang in translation_models: | |
| translated = translation_models[target_lang](english_caption)[0]['translation_text'] | |
| else: | |
| translated = "Translation not available" | |
| # Step 3: Generate Speech (English caption for now) | |
| tts = gTTS(english_caption, lang="en") | |
| tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") | |
| tts.save(tmp_file.name) | |
| return english_caption, translated, tmp_file.name | |
| # ---------------------- | |
| # VQA | |
| # ---------------------- | |
| def vqa_answer(image, question): | |
| inputs = vqa_processor(image, question, return_tensors="pt").to(device) | |
| with torch.no_grad(): | |
| out = vqa_model.generate(**inputs, max_new_tokens=50) | |
| answer = vqa_processor.decode(out[0], skip_special_tokens=True) | |
| # Run safety filter on answers too | |
| if not is_caption_safe(answer): | |
| return "⚠️ Warning: Unsafe or inappropriate content detected!" | |
| return answer | |
| # ---------------------- | |
| # Gradio UI | |
| # ---------------------- | |
| with gr.Blocks(title="BLIP Vision App") as demo: | |
| gr.Markdown("## 🖼️ BLIP: Image Captioning + Translation + Speech + VQA (with Safety Filter)") | |
| with gr.Tab("Caption + Translate + Speak"): | |
| with gr.Row(): | |
| img_in = gr.Image(type="pil", label="Upload Image") | |
| lang_in = gr.Dropdown(["Hindi", "French", "Spanish"], label="Translate To", value="Hindi") | |
| eng_out = gr.Textbox(label="English Caption") | |
| trans_out = gr.Textbox(label="Translated Caption") | |
| audio_out = gr.Audio(label="Spoken Caption", type="filepath") | |
| btn1 = gr.Button("Generate Caption, Translate & Speak") | |
| btn1.click(generate_caption_translate_speak, inputs=[img_in, lang_in], outputs=[eng_out, trans_out, audio_out]) | |
| with gr.Tab("Visual Question Answering (VQA)"): | |
| with gr.Row(): | |
| img_vqa = gr.Image(type="pil", label="Upload Image") | |
| q_in = gr.Textbox(label="Ask a Question about the Image") | |
| ans_out = gr.Textbox(label="Answer") | |
| btn2 = gr.Button("Ask") | |
| btn2.click(vqa_answer, inputs=[img_vqa, q_in], outputs=ans_out) | |
| demo.launch() | |
| # import gradio as gr | |
| # from transformers import ( | |
| # BlipProcessor, | |
| # BlipForConditionalGeneration, | |
| # BlipForQuestionAnswering, | |
| # pipeline | |
| # ) | |
| # from PIL import Image | |
| # import torch | |
| # from gtts import gTTS | |
| # import tempfile | |
| # # ---------------------- | |
| # # Device setup | |
| # # ---------------------- | |
| # device = "cuda" if torch.cuda.is_available() else "cpu" | |
| # # ---------------------- | |
| # # Load Models Once | |
| # # ---------------------- | |
| # print("🔄 Loading models...") | |
| # # Captioning | |
| # caption_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large") | |
| # caption_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large").to(device) | |
| # # VQA | |
| # vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") | |
| # vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base").to(device) | |
| # # Translation | |
| # translation_models = { | |
| # "Hindi": pipeline("translation", model="Helsinki-NLP/opus-mt-en-hi"), | |
| # "French": pipeline("translation", model="Helsinki-NLP/opus-mt-en-fr"), | |
| # "Spanish": pipeline("translation", model="Helsinki-NLP/opus-mt-en-es"), | |
| # } | |
| # # Safety Moderation Pipeline | |
| # moderation_model = pipeline("text-classification", model="unitary/toxic-bert") | |
| # print("✅ All models loaded!") | |
| # # ---------------------- | |
| # # Safety Filter Function | |
| # # ---------------------- | |
| # def is_caption_safe(caption): | |
| # try: | |
| # votes = moderation_model(caption) | |
| # # If return_all_scores=True, it's [[{label, score}, ...]] | |
| # if isinstance(votes, list) and isinstance(votes[0], list): | |
| # votes = votes[0] | |
| # # Loop through scores | |
| # for item in votes: | |
| # if isinstance(item, dict) and item.get("label") in ["V", "V2"] and item.get("score", 0) > 0.5: | |
| # return False | |
| # except Exception as e: | |
| # print("⚠️ Moderation failed:", e) | |
| # # Fallback keyword check | |
| # unsafe_keywords = [ | |
| # "gun", "blood", "skull", "kill", "corpse", "gore", "knife", "weapon", "fire", | |
| # "murder", "dead", "death", "suicide", "bomb", "explosion", "terrorist", "assault", | |
| # "stab", "shoot", "pistol", "rifle", "shotgun", "grenade", "horror", "beheaded", | |
| # "torture", "hostage", "rape", "war", "massacre", "chainsaw", "poison", "strangle", | |
| # "hang", "drown" | |
| # ] | |
| # if any(word in caption.lower() for word in unsafe_keywords): | |
| # return False | |
| # return True | |
| # # ---------------------- | |
| # # Caption + Translate + Speak | |
| # # ---------------------- | |
| # def generate_caption_translate_speak(image, target_lang): | |
| # # Step 1: Caption | |
| # inputs = caption_processor(images=image, return_tensors="pt").to(device) | |
| # with torch.no_grad(): | |
| # out = caption_model.generate(**inputs, max_new_tokens=50) | |
| # english_caption = caption_processor.decode(out[0], skip_special_tokens=True) | |
| # # Step 1.5: Safety Check | |
| # if not is_caption_safe(english_caption): | |
| # return "⚠️ Warning: Unsafe or inappropriate content detected!", "", None | |
| # # Step 2: Translate | |
| # if target_lang in translation_models: | |
| # translated = translation_models[target_lang](english_caption)[0]['translation_text'] | |
| # else: | |
| # translated = "Translation not available" | |
| # # Step 3: Generate Speech (English caption for now) | |
| # tts = gTTS(english_caption, lang="en") | |
| # tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") | |
| # tts.save(tmp_file.name) | |
| # return english_caption, translated, tmp_file.name | |
| # # ---------------------- | |
| # # VQA | |
| # # ---------------------- | |
| # def vqa_answer(image, question): | |
| # inputs = vqa_processor(image, question, return_tensors="pt").to(device) | |
| # with torch.no_grad(): | |
| # out = vqa_model.generate(**inputs, max_new_tokens=50) | |
| # answer = vqa_processor.decode(out[0], skip_special_tokens=True) | |
| # # Safety filter | |
| # if not is_caption_safe(answer): | |
| # return "⚠️ Warning: Unsafe or inappropriate content detected!" | |
| # return answer | |
| # # ---------------------- | |
| # # Gradio UI | |
| # # ---------------------- | |
| # with gr.Blocks(title="BLIP Vision App") as demo: | |
| # gr.Markdown("## 🖼️ BLIP: Image Captioning + Translation + Speech + VQA (with Safety Filter)") | |
| # with gr.Tab("Caption + Translate + Speak"): | |
| # with gr.Row(): | |
| # img_in = gr.Image(type="pil", label="Upload Image") | |
| # lang_in = gr.Dropdown(["Hindi", "French", "Spanish"], label="Translate To", value="Hindi") | |
| # eng_out = gr.Textbox(label="English Caption") | |
| # trans_out = gr.Textbox(label="Translated Caption") | |
| # audio_out = gr.Audio(label="Spoken Caption", type="filepath") | |
| # btn1 = gr.Button("Generate Caption, Translate & Speak") | |
| # btn1.click(generate_caption_translate_speak, inputs=[img_in, lang_in], outputs=[eng_out, trans_out, audio_out]) | |
| # with gr.Tab("Visual Question Answering (VQA)"): | |
| # with gr.Row(): | |
| # img_vqa = gr.Image(type="pil", label="Upload Image") | |
| # q_in = gr.Textbox(label="Ask a Question about the Image") | |
| # ans_out = gr.Textbox(label="Answer") | |
| # btn2 = gr.Button("Ask") | |
| # btn2.click(vqa_answer, inputs=[img_vqa, q_in], outputs=ans_out) | |
| # demo.launch() | |