update files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,15 @@
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+pyvenv.cfg
+share/
+bin/
+
+# PyCharm files
+.idea/
+.cursorignore

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.4
+    hooks:
+    -   id: ruff
+        args: [ --fix ]
+    -   id: ruff-format

README.md

@@ -9,5 +9,5 @@ app_file: app.py
 pinned: false
 license: mit
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+ 
+# Multi-modality Misogyny Moderation For Vividhata

app.py

@@ -0,0 +1,201 @@
+import base64
+from io import BytesIO
+from pathlib import Path
+from urllib.parse import urlparse
+
+import dotenv
+import gradio as gr
+import requests
+from clients import get_client_module
+from hf_datasets import dataset_rootdir
+from omegaconf import DictConfig, OmegaConf
+from PIL import Image
+from prompts import get_prompt_module
+
+
+dotenv.load_dotenv()
+
+prompt_versions = [d.stem for d in Path("./prompts").iterdir() if d.is_file() and not d.name.startswith("_")]
+
+
+class ConfigManager:
+    def __init__(self):
+        self.configs: dict = {}  # internal configs for all models
+        self.ignore_keys = ["type", "client_name", "model_name"]
+
+        # initialize configs
+        self.update()
+
+    def update(self):
+        """Reload configs"""
+        self.configs.clear()  # remove cache
+
+        # reload API-based models
+        configs = OmegaConf.load("./model/api.yaml")
+        configs = {key: configs[key] for key in configs if key not in self.ignore_keys}
+        self.configs.update(configs)
+
+        # reload HF-based models
+        configs = OmegaConf.load("./model/hf.yaml")
+        configs = {key: configs[key] for key in configs if key not in self.ignore_keys}
+        self.configs.update({"huggingface": DictConfig(configs)})
+
+    def clients(self):
+        """Display all available clients"""
+        return list(self.configs.keys())
+
+    def models(self, client=None):
+        if client is None:
+            client = self.clients()[0]
+        return list(self.configs[client].available_models)
+
+
+config_manager = ConfigManager()
+
+
+def link_client_and_model(client, model):  # noqa
+    all_models = config_manager.models(client)
+    return gr.Dropdown(choices=all_models, value=all_models[0])
+
+
+def display_prompt(prompt_version):
+    prompt_module = get_prompt_module(prompt_version)
+    description = prompt_module.description()
+    return description
+
+
+def encode_image(image):
+    buffered = BytesIO()
+    image.save(buffered, format="PNG")
+    return base64.b64encode(buffered.getvalue()).decode("utf-8")
+
+
+def load_image(image_url_or_path, timeout=None):
+    result = urlparse(image_url_or_path)
+    if result.scheme in ("http", "https") and result.netloc and result.path:
+        image = Image.open(BytesIO(requests.get(image_url_or_path, timeout=timeout).content))
+
+    elif Path(image_url_or_path).is_file():
+        image = Image.open(image_url_or_path)
+    else:
+        if image_url_or_path.startswith("data:image/"):
+            image_url_or_path = image_url_or_path.split(",")[1]
+
+        # Try to load as base64
+        try:
+            base64_image = base64.decodebytes(image_url_or_path.encode())
+            image = Image.open(BytesIO(base64_image))
+
+        except Exception:
+            raise gr.Error(
+                "Incorrect image source. Must be a valid URL starting with `http://` or `https://`, "
+                "a valid path to an image file, or a base64 encoded string."
+            )
+    return image
+
+
+def llm_analyse(client, model, api_key, image, prompt):
+    try:
+        prompt_module = get_prompt_module(prompt)
+        client_module = get_client_module(client)
+
+        base64_image = f"data:image/png;base64,{encode_image(image)}"
+
+        if api_key == "":
+            api_key = None
+
+        result = client_module.sync_generate(base64_image, prompt_module.messages_encoder, model, api_key=api_key)
+        return result
+
+    except Exception as e:
+        return gr.Error(f"Error processing image: {e}")
+
+
+with gr.Blocks(
+    theme=gr.themes.Default(primary_hue="orange"),
+    css="""
+    #app-container { max-width: 1400px; margin: auto; padding: 10px; }
+    #title { text-align: center; margin-bottom: 10px; font-size: 24px; }
+    #groq-badge { text-align: center; margin-top: 10px; }
+    .gr-button { border-radius: 15px; }
+    .gr-input, .gr-box { border-radius: 10px; }
+    .gr-form { gap: 5px; }
+    .gr-block.gr-box { padding: 10px; }
+    .gr-paddle { height: auto; }
+    """,
+) as demo:
+    gr.Markdown("# Image Moderation WebUI", elem_id="title")
+
+    # --------------- Client and Model Selection Block --------------- #
+    with gr.Row(equal_height=True):
+        with gr.Column(scale=3):
+            prompt_version_input = gr.Dropdown(
+                prompt_versions,
+                value="-- Please Select --",
+                allow_custom_value=True,
+                label="Choose Prompt:",
+            )
+
+            client_input = gr.Dropdown(
+                config_manager.clients(), label="Choose Client:", info="HuggingFace Requires a GPU"
+            )
+
+            model_input = gr.Dropdown(config_manager.models(), label="Choose Model:")
+
+            api_input = gr.Textbox(
+                type="password",
+                label="API Key:",
+                info="Leave this field blank to use the default key, or if you are using HuggingFace",
+            )
+
+            image_input = gr.Image(type="pil", label="Upload Image:", height=300, sources=["upload"])
+            url_input = gr.Textbox(
+                label="or Paste Image URL, Local File Path, or Base64 String:",
+                info="Press Enter to load the image",
+                lines=1,
+            )
+
+            with gr.Row():
+                with gr.Column(scale=1, min_width=160):
+                    pos_button = gr.Button("👍 Positive Demo")
+                with gr.Column(scale=1, min_width=160):
+                    neg_button = gr.Button("👎 Negative Demo")
+
+        with gr.Column(scale=5):
+            prompt_text_input = gr.Textbox(label="or Paste Prompt Here:", lines=18)
+            model_output = gr.Textbox(label="Model Output:", lines=18)
+
+            with gr.Row():
+                with gr.Column(scale=1, min_width=120):
+                    analyze_button = gr.Button("🚀 Analyze Image", variant="primary")
+                with gr.Column(scale=1, min_width=120):
+                    clean_button = gr.Button("🧹 Clean Output", variant="primary")
+
+    client_input.change(fn=link_client_and_model, inputs=[client_input, model_input], outputs=model_input)
+
+    prompt_version_input.input(fn=display_prompt, inputs=prompt_version_input, outputs=prompt_text_input)
+
+    clean_button.click(fn=lambda: gr.Textbox(value=""), inputs=None, outputs=model_output)
+
+    url_input.submit(fn=load_image, inputs=url_input, outputs=image_input)
+
+    pos_button.click(
+        fn=lambda: load_image(Path(dataset_rootdir, "semeval2022/demo-pos.jpg").as_posix()),
+        inputs=None,
+        outputs=image_input,
+    )
+    neg_button.click(
+        fn=lambda: load_image(Path(dataset_rootdir, "semeval2022/demo-neg.jpg").as_posix()),
+        inputs=None,
+        outputs=image_input,
+    )
+
+    # ------------------------- Image Analysis Block ------------------------- #
+    analyze_button.click(
+        fn=llm_analyse,
+        inputs=[client_input, model_input, api_input, image_input, prompt_version_input],
+        outputs=model_output,
+    )
+
+
+demo.launch(share=False)

config.yaml

@@ -0,0 +1,31 @@
+# hydra/cli specific settings
+hydra:
+  run:
+    # where to store run results
+    dir: outputs/${dataset_name}-${dataset_split}/${model.client_name}-${model.model_name}/prompt-${prompt_version}-${now:%y%m%d_%H%M%S}
+  output_subdir: null
+  job:
+    # change the working directory to the run directory
+    chdir: false
+  sweep:
+    dir: multirun
+    # change the working directory to the run directory
+    subdir: ${hydra.job.override_dirname}
+
+defaults:
+  # can be hf or api
+  - model: ???
+  # for hydra 1.1 compatibility
+  - _self_
+
+prompt_version: v3
+
+dataset_name: semeval2022
+dataset_split: validation
+
+# if batch_mode is set to false, standard asynchronous inference will be used.
+# if batch_mode is set to true, then:
+#   1) if batch_job_ids are provided, their corresponding results will be fetched and concatenated as the prediction output.
+#   2) if batch_job_ids are not provided, a batched inference job will be executed and new job ids will be produced.
+batch_mode: false
+batch_job_ids:

demo_webui.py

@@ -0,0 +1,200 @@
+import base64
+from io import BytesIO
+from pathlib import Path
+from urllib.parse import urlparse
+
+import dotenv
+import gradio as gr
+import requests
+from clients import get_client_module
+from omegaconf import DictConfig, OmegaConf
+from PIL import Image
+from prompts import get_prompt_module
+
+
+dotenv.load_dotenv()
+
+prompt_versions = [d.stem for d in Path("./prompts").iterdir() if d.is_file() and not d.name.startswith("_")]
+
+
+class ConfigManager:
+    def __init__(self):
+        self.configs: dict = {}  # internal configs for all models
+        self.ignore_keys = ["type", "client_name", "model_name"]
+
+        # initialize configs
+        self.update()
+
+    def update(self):
+        """Reload configs"""
+        self.configs.clear()  # remove cache
+
+        # reload API-based models
+        configs = OmegaConf.load("./model/api.yaml")
+        configs = {key: configs[key] for key in configs if key not in self.ignore_keys}
+        self.configs.update(configs)
+
+        # reload HF-based models
+        configs = OmegaConf.load("./model/hf.yaml")
+        configs = {key: configs[key] for key in configs if key not in self.ignore_keys}
+        self.configs.update({"huggingface": DictConfig(configs)})
+
+    def clients(self):
+        """Display all available clients"""
+        return list(self.configs.keys())
+
+    def models(self, client=None):
+        if client is None:
+            client = self.clients()[0]
+        return list(self.configs[client].available_models)
+
+
+config_manager = ConfigManager()
+
+
+def link_client_and_model(client, model):  # noqa
+    all_models = config_manager.models(client)
+    return gr.Dropdown(choices=all_models, value=all_models[0])
+
+
+def display_prompt(prompt_version):
+    prompt_module = get_prompt_module(prompt_version)
+    description = prompt_module.description()
+    return description
+
+
+def encode_image(image):
+    buffered = BytesIO()
+    image.save(buffered, format="PNG")
+    return base64.b64encode(buffered.getvalue()).decode("utf-8")
+
+
+def load_image(image_url_or_path, timeout=None):
+    result = urlparse(image_url_or_path)
+    if result.scheme in ("http", "https") and result.netloc and result.path:
+        image = Image.open(BytesIO(requests.get(image_url_or_path, timeout=timeout).content))
+
+    elif Path(image_url_or_path).is_file():
+        image = Image.open(image_url_or_path)
+    else:
+        if image_url_or_path.startswith("data:image/"):
+            image_url_or_path = image_url_or_path.split(",")[1]
+
+        # Try to load as base64
+        try:
+            base64_image = base64.decodebytes(image_url_or_path.encode())
+            image = Image.open(BytesIO(base64_image))
+
+        except Exception:
+            raise gr.Error(
+                "Incorrect image source. Must be a valid URL starting with `http://` or `https://`, "
+                "a valid path to an image file, or a base64 encoded string."
+            )
+    return image
+
+
+def llm_analyse(client, model, api_key, image, prompt):
+    try:
+        prompt_module = get_prompt_module(prompt)
+        client_module = get_client_module(client)
+
+        base64_image = f"data:image/png;base64,{encode_image(image)}"
+
+        if api_key == "":
+            api_key = None
+
+        result = client_module.sync_generate(base64_image, prompt_module.messages_encoder, model, api_key=api_key)
+        return result
+
+    except Exception as e:
+        return gr.Error(f"Error processing image: {e}")
+
+
+with gr.Blocks(
+    theme=gr.themes.Default(primary_hue="orange"),
+    css="""
+    #app-container { max-width: 1400px; margin: auto; padding: 10px; }
+    #title { text-align: center; margin-bottom: 10px; font-size: 24px; }
+    #groq-badge { text-align: center; margin-top: 10px; }
+    .gr-button { border-radius: 15px; }
+    .gr-input, .gr-box { border-radius: 10px; }
+    .gr-form { gap: 5px; }
+    .gr-block.gr-box { padding: 10px; }
+    .gr-paddle { height: auto; }
+    """,
+) as demo:
+    gr.Markdown("# Image Moderation WebUI", elem_id="title")
+
+    # --------------- Client and Model Selection Block --------------- #
+    with gr.Row(equal_height=True):
+        with gr.Column(scale=3):
+            prompt_version_input = gr.Dropdown(
+                prompt_versions,
+                value="-- Please Select --",
+                allow_custom_value=True,
+                label="Choose Prompt:",
+            )
+
+            client_input = gr.Dropdown(
+                config_manager.clients(), label="Choose Client:", info="HuggingFace Requires a GPU"
+            )
+
+            model_input = gr.Dropdown(config_manager.models(), label="Choose Model:")
+
+            api_input = gr.Textbox(
+                type="password",
+                label="API Key:",
+                info="Leave this field blank to use the default key, or if you are using HuggingFace",
+            )
+
+            image_input = gr.Image(type="pil", label="Upload Image:", height=300, sources=["upload"])
+            url_input = gr.Textbox(
+                label="or Paste Image URL, Local File Path, or Base64 String:",
+                info="Press Enter to load the image",
+                lines=1,
+            )
+
+            with gr.Row():
+                with gr.Column(scale=1, min_width=160):
+                    pos_button = gr.Button("👍 Positive Demo")
+                with gr.Column(scale=1, min_width=160):
+                    neg_button = gr.Button("👎 Negative Demo")
+
+        with gr.Column(scale=5):
+            prompt_text_input = gr.Textbox(label="or Paste Prompt Here:", lines=18)
+            model_output = gr.Textbox(label="Model Output:", lines=18)
+
+            with gr.Row():
+                with gr.Column(scale=1, min_width=120):
+                    analyze_button = gr.Button("🚀 Analyze Image", variant="primary")
+                with gr.Column(scale=1, min_width=120):
+                    clean_button = gr.Button("🧹 Clean Output", variant="primary")
+
+    client_input.change(fn=link_client_and_model, inputs=[client_input, model_input], outputs=model_input)
+
+    prompt_version_input.input(fn=display_prompt, inputs=prompt_version_input, outputs=prompt_text_input)
+
+    clean_button.click(fn=lambda: gr.Textbox(value=""), inputs=None, outputs=model_output)
+
+    url_input.submit(fn=load_image, inputs=url_input, outputs=image_input)
+
+    pos_button.click(
+        fn=lambda: load_image(Path("./demo-pos.jpg").as_posix()),
+        inputs=None,
+        outputs=image_input,
+    )
+    neg_button.click(
+        fn=lambda: load_image(Path("./demo-neg.jpg").as_posix()),
+        inputs=None,
+        outputs=image_input,
+    )
+
+    # ------------------------- Image Analysis Block ------------------------- #
+    analyze_button.click(
+        fn=llm_analyse,
+        inputs=[client_input, model_input, api_input, image_input, prompt_version_input],
+        outputs=model_output,
+    )
+
+
+demo.launch(share=False)

model/api.yaml

@@ -0,0 +1,45 @@
+type: API
+
+client_name: openai
+model_name: ${model.${model.client_name}.model}
+
+openai:
+  model: "gpt-4o-mini"
+  api_key: ${oc.env:OPENAI_API_KEY}
+  available_models:
+    # 0.000425, fixed,
+    - gpt-4o-mini
+    # 0.000213, fixed
+    - gpt-4o
+    # 0.000098, max
+    - gpt-4.1-nano
+    # 0.000259, max
+    - gpt-4.1-mini
+    # 0.00017, fixed
+    - gpt-4.1
+
+together:
+  model: "meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo"
+  api_key: ${oc.env:TOGETHER_API_KEY}
+  available_models:
+    - "meta-llama/Llama-Vision-Free"
+    - "meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo"
+    - "meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo"
+
+    # moderation models
+    - "meta-llama/Llama-Guard-3-11B-Vision-Turbo"
+
+
+
+groq:
+  client:
+    _target_: groq.Groq
+
+  async_client:
+    _target_: groq.AsyncGroq
+
+  api_key: ${oc.env:GROQ_API_KEY}
+
+  model: "meta-llama/llama-4-scout-17b-16e-instruct"
+  available_models:
+    - "meta-llama/llama-4-scout-17b-16e-instruct"

model/hf.yaml

@@ -0,0 +1,25 @@
+type: HF
+
+client_name: huggingface
+
+model_name: unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit
+
+available_models:
+  - unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit
+
+model_loader:
+  _target_: transformers.AutoModelForImageTextToText.from_pretrained
+  pretrained_model_name_or_path: ${model.model_name}
+  device_map: auto
+  trust_remote_code: true
+
+processor_loader:
+  _target_: transformers.AutoProcessor.from_pretrained
+  pretrained_model_name_or_path: ${model.model_name}
+  trust_remote_code: true
+
+
+
+
+
+

pyproject.toml

@@ -0,0 +1,40 @@
+[tool.ruff]
+line-length = 119
+
+[tool.ruff.lint]
+select = [
+    "C",  # flake8-comprehensions
+    "E",  # pycodestyle-error
+    "W",  # pycodestyle-warning
+    "F",  # Pyflakes
+    "I",  # isort
+]
+ignore = [
+    "C901",  # complex-structure
+    "E402",  # module-import-not-at-top-of-file
+    "E501",  # line-too-long
+    "E741",  # ambiguous-variable-name
+]
+
+
+# Ignore import violations in all `__init__.py` files.
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = [
+    "F403",  # undefined-local-with-import-star
+]
+
+[tool.ruff.lint.isort]
+lines-after-imports = 2
+
+[tool.ruff.format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"