Spaces:
Sleeping
Sleeping
| # import gradio as gr | |
| # from transformers import BlipProcessor, BlipForConditionalGeneration | |
| # from gtts import gTTS | |
| # import io | |
| # from PIL import Image | |
| # # ------------------------------- | |
| # # Load BLIP-base model (lighter version) | |
| # # ------------------------------- | |
| # processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large") | |
| # model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large") | |
| # # ------------------------------- | |
| # # Generate caption function | |
| # # ------------------------------- | |
| # # def generate_caption_tts(image): | |
| # # caption = generate_caption(model, processor, image) | |
| # # audio_file = text_to_audio_file(caption) | |
| # # return caption, audio_file # return file path, not BytesIO | |
| # # ------------------------------- | |
| # # Convert text to speech using gTTS | |
| # # ------------------------------- | |
| # import tempfile | |
| # import pyttsx3 | |
| # def text_to_audio_file(text): | |
| # # Create a temporary file | |
| # tmp_file = tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) | |
| # tmp_path = tmp_file.name | |
| # tmp_file.close() | |
| # engine = pyttsx3.init() | |
| # engine.save_to_file(text, tmp_path) | |
| # engine.runAndWait() | |
| # return tmp_path | |
| # def generate_caption_from_image(model, processor, image): | |
| # # image: PIL.Image | |
| # inputs = processor(images=image, return_tensors="pt") | |
| # out = model.generate(**inputs) | |
| # caption = processor.decode(out[0], skip_special_tokens=True) | |
| # return caption | |
| # # ------------------------------- | |
| # # Gradio interface: Caption + Audio | |
| # # ------------------------------- | |
| # def generate_caption_tts(image): | |
| # caption = generate_caption_from_image(model, processor, image) # uses global model/processor | |
| # # audio_file = text_to_audio_file(caption) | |
| # return caption | |
| # interface = gr.Interface( | |
| # fn=generate_caption_tts, | |
| # inputs=gr.Image(type="numpy"), | |
| # outputs=[gr.Textbox(label="Generated Caption")], | |
| # title="Image Captioning for Visually Impaired", | |
| # description="Upload an image, get a caption and audio description." | |
| # ) | |
| # interface.launch() | |
| # # demo.launch(share=True) | |
| # import gradio as gr | |
| # from transformers import ( | |
| # BlipProcessor, | |
| # BlipForConditionalGeneration, | |
| # BlipForQuestionAnswering, | |
| # pipeline | |
| # ) | |
| # moderation_model = pipeline( | |
| # "text-classification", | |
| # model="Vrandan/Comment-Moderation", | |
| # return_all_scores=True | |
| # ) | |
| # from PIL import Image | |
| # import torch | |
| # from gtts import gTTS | |
| # import tempfile | |
| # # ---------------------- | |
| # # Device setup | |
| # # ---------------------- | |
| # device = "cuda" if torch.cuda.is_available() else "cpu" | |
| # # ---------------------- | |
| # # Load Models Once | |
| # # ---------------------- | |
| # print("🔄 Loading models...") | |
| # # Captioning | |
| # caption_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large") | |
| # caption_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large").to(device) | |
| # # VQA | |
| # vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") | |
| # vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base").to(device) | |
| # # Translation | |
| # translation_models = { | |
| # "Hindi": pipeline("translation", model="Helsinki-NLP/opus-mt-en-hi"), | |
| # "French": pipeline("translation", model="Helsinki-NLP/opus-mt-en-fr"), | |
| # "Spanish": pipeline("translation", model="Helsinki-NLP/opus-mt-en-es"), | |
| # } | |
| # # Safety Moderation Pipeline | |
| # moderation_model = pipeline("text-classification", model="unitary/toxic-bert") | |
| # print("✅ All models loaded!") | |
| # # ---------------------- | |
| # # Safety Filter Function | |
| # # ---------------------- | |
| # def is_caption_safe(caption): | |
| # try: | |
| # votes = moderation_model(caption) | |
| # # If return_all_scores=True, it's [[{label, score}, ...]] | |
| # if isinstance(votes, list) and isinstance(votes[0], list): | |
| # votes = votes[0] | |
| # # Now safe to loop | |
| # for item in votes: | |
| # if isinstance(item, dict) and item.get("label") in ["V", "V2"] and item.get("score", 0) > 0.5: | |
| # return False | |
| # except Exception as e: | |
| # print("⚠️ Moderation failed:", e) | |
| # # Fallback keywords | |
| # unsafe_keywords = [ | |
| # "gun", "blood", "skull", "kill", "corpse", "gore", "knife", "weapon", | |
| # "fire", "murder", "dead", "death", "suicide", "bomb", "explosion", | |
| # "terrorist", "assault", "stab", "shoot", "pistol", "rifle", "shotgun", | |
| # "grenade", "horror", "beheaded", "torture", "hostage", "rape", | |
| # "war", "massacre", "chainsaw", "poison", "strangle", "hang", "drown" | |
| # ] | |
| # if any(word in caption.lower() for word in unsafe_keywords): | |
| # return False | |
| # return True | |
| # # ---------------------- | |
| # # Caption + Translate + Speak | |
| # # ---------------------- | |
| # def generate_caption_translate_speak(image, target_lang): | |
| # # Step 1: Caption | |
| # inputs = caption_processor(images=image, return_tensors="pt").to(device) | |
| # with torch.no_grad(): | |
| # out = caption_model.generate(**inputs, max_new_tokens=50) | |
| # english_caption = caption_processor.decode(out[0], skip_special_tokens=True) | |
| # # Step 1.5: Safety Check | |
| # if not is_caption_safe(english_caption): | |
| # return "⚠️ Warning: Unsafe or inappropriate content detected!", "", None | |
| # # Step 2: Translate | |
| # if target_lang in translation_models: | |
| # translated = translation_models[target_lang](english_caption)[0]['translation_text'] | |
| # else: | |
| # translated = "Translation not available" | |
| # # Step 3: Generate Speech (English caption for now) | |
| # tts = gTTS(english_caption, lang="en") | |
| # tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") | |
| # tts.save(tmp_file.name) | |
| # return english_caption, translated, tmp_file.name | |
| # # ---------------------- | |
| # # VQA | |
| # # ---------------------- | |
| # def vqa_answer(image, question): | |
| # inputs = vqa_processor(image, question, return_tensors="pt").to(device) | |
| # with torch.no_grad(): | |
| # out = vqa_model.generate(**inputs, max_new_tokens=50) | |
| # answer = vqa_processor.decode(out[0], skip_special_tokens=True) | |
| # # Run safety filter on answers too | |
| # if not is_caption_safe(answer): | |
| # return "⚠️ Warning: Unsafe or inappropriate content detected!" | |
| # return answer | |
| # # ---------------------- | |
| # # Gradio UI | |
| # # ---------------------- | |
| # with gr.Blocks(title="BLIP Vision App") as demo: | |
| # gr.Markdown("## 🖼️ BLIP: Image Captioning + Translation + Speech + VQA (with Safety Filter)") | |
| # with gr.Tab("Caption + Translate + Speak"): | |
| # with gr.Row(): | |
| # img_in = gr.Image(type="pil", label="Upload Image") | |
| # lang_in = gr.Dropdown(["Hindi", "French", "Spanish"], label="Translate To", value="Hindi") | |
| # eng_out = gr.Textbox(label="English Caption") | |
| # trans_out = gr.Textbox(label="Translated Caption") | |
| # audio_out = gr.Audio(label="Spoken Caption", type="filepath") | |
| # btn1 = gr.Button("Generate Caption, Translate & Speak") | |
| # btn1.click(generate_caption_translate_speak, inputs=[img_in, lang_in], outputs=[eng_out, trans_out, audio_out]) | |
| # with gr.Tab("Visual Question Answering (VQA)"): | |
| # with gr.Row(): | |
| # img_vqa = gr.Image(type="pil", label="Upload Image") | |
| # q_in = gr.Textbox(label="Ask a Question about the Image") | |
| # ans_out = gr.Textbox(label="Answer") | |
| # btn2 = gr.Button("Ask") | |
| # btn2.click(vqa_answer, inputs=[img_vqa, q_in], outputs=ans_out) | |
| # demo.launch() | |
| import gradio as gr | |
| from transformers import ( | |
| BlipProcessor, | |
| BlipForConditionalGeneration, | |
| BlipForQuestionAnswering, | |
| pipeline | |
| ) | |
| from PIL import Image | |
| import torch | |
| import tempfile | |
| import base64 | |
| # ---------------------- | |
| # Device | |
| # ---------------------- | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| # ---------------------- | |
| # Simple BEEP sound (base64) | |
| # ---------------------- | |
| BEEP_BASE64 = """ | |
| SUQzAwAAAAAAF1RTU0UAAAAPAAADTGF2ZjU4LjMyLjEwNAAAAAAAAAAAAAAA//uQxAADB... | |
| """ | |
| # Convert base64 to temp mp3 file | |
| def load_beep(): | |
| audio_bytes = base64.b64decode(BEEP_BASE64) | |
| tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") | |
| tmp.write(audio_bytes) | |
| tmp.close() | |
| return tmp.name | |
| # ---------------------- | |
| # Load models | |
| # ---------------------- | |
| print("🔄 Loading models...") | |
| caption_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large") | |
| caption_model = BlipForConditionalGeneration.from_pretrained( | |
| "Salesforce/blip-image-captioning-large" | |
| ).to(device) | |
| vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") | |
| vqa_model = BlipForQuestionAnswering.from_pretrained( | |
| "Salesforce/blip-vqa-base" | |
| ).to(device) | |
| translation_models = { | |
| "Hindi": pipeline("translation", model="Helsinki-NLP/opus-mt-en-hi"), | |
| "French": pipeline("translation", model="Helsinki-NLP/opus-mt-en-fr"), | |
| "Spanish": pipeline("translation", model="Helsinki-NLP/opus-mt-en-es"), | |
| } | |
| moderation_model = pipeline("text-classification", model="unitary/toxic-bert") | |
| print("✅ All models loaded!") | |
| # ---------------------- | |
| # Safety check | |
| # ---------------------- | |
| def is_caption_safe(caption): | |
| try: | |
| result = moderation_model(caption) | |
| if isinstance(result, list) and "label" in result[0]: | |
| if result[0]["label"] == "toxic" and result[0]["score"] > 0.5: | |
| return False | |
| except: | |
| pass | |
| unsafe_words = ["gun", "kill", "dead", "weapon", "blood"] | |
| return not any(w in caption.lower() for w in unsafe_words) | |
| # ---------------------- | |
| # Auto Caption + Translate + BEEP | |
| # ---------------------- | |
| def auto_process(image, target_lang): | |
| if image is None: | |
| return "", "", None | |
| # Caption | |
| inputs = caption_processor(images=image, return_tensors="pt").to(device) | |
| with torch.no_grad(): | |
| output = caption_model.generate(**inputs, max_new_tokens=40) | |
| caption = caption_processor.decode(output[0], skip_special_tokens=True) | |
| # Safety | |
| if not is_caption_safe(caption): | |
| return "⚠️ Unsafe content detected!", "", load_beep() | |
| # Translate | |
| translated = translation_models[target_lang](caption)[0]["translation_text"] | |
| # Always play BEEP once caption is ready | |
| beep_file = load_beep() | |
| return caption, translated, beep_file | |
| # ---------------------- | |
| # VQA | |
| # ---------------------- | |
| def vqa_answer(image, question): | |
| if image is None or not question: | |
| return "" | |
| inputs = vqa_processor(image, question, return_tensors="pt").to(device) | |
| with torch.no_grad(): | |
| out = vqa_model.generate(**inputs, max_new_tokens=30) | |
| ans = vqa_processor.decode(out[0], skip_special_tokens=True) | |
| if not is_caption_safe(ans): | |
| return "⚠️ Unsafe content detected!" | |
| return ans | |
| # ---------------------- | |
| # UI | |
| # ---------------------- | |
| with gr.Blocks(title="BLIP App") as demo: | |
| gr.Markdown("## 🖼️ Auto-Caption + Translation + Automatic Beep") | |
| with gr.Tab("Auto Caption"): | |
| img = gr.Image(type="pil", label="Upload Image") | |
| lang = gr.Dropdown(["Hindi", "French", "Spanish"], value="Hindi", label="Translate To") | |
| out_eng = gr.Textbox(label="English Caption") | |
| out_trans = gr.Textbox(label="Translated") | |
| out_audio = gr.Audio(label="Beep", autoplay=True) | |
| # 🔥 Auto-run when image is uploaded | |
| img.change(auto_process, inputs=[img, lang], outputs=[out_eng, out_trans, out_audio]) | |
| lang.change(auto_process, inputs=[img, lang], outputs=[out_eng, out_trans, out_audio]) | |
| with gr.Tab("VQA"): | |
| img_vqa = gr.Image(type="pil") | |
| q = gr.Textbox(label="Ask a question") | |
| ans = gr.Textbox(label="Answer") | |
| ask_btn = gr.Button("Ask") | |
| ask_btn.click(vqa_answer, inputs=[img_vqa, q], outputs=ans) | |
| demo.launch() | |