Update

README.md

@@ -1,13 +1,15 @@
 ---
-title: IBYDMT
-emoji: 💻
-colorFrom: green
-colorTo: red
+title: I Bet You Did Not Mean That
+emoji: 🔎
+colorFrom: blue
+colorTo: indigo
 sdk: streamlit
-sdk_version: 1.35.0
+sdk_version: 1.25.0
 app_file: app.py
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+datasets:
+    - jacopoteneggi/IBYDMT
+preload_from_hub:
+    - laion/CLIP-ViT-B-32-laion2B-s34B-b79K open_clip_config.json,open_clip_pytorch_model.bin
+pinned: true
+license: cc-by-nc-4.0
+---

app.py

@@ -0,0 +1,54 @@
+import numpy as np
+import open_clip
+import streamlit as st
+import torch
+
+from app_lib.main import main
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+st.set_page_config(
+    layout="wide",
+    initial_sidebar_state=st.session_state.get("sidebar_state", "collapsed"),
+)
+st.session_state.sidebar_state = "collapsed"
+st.markdown(
+    """
+        <style>
+            textarea {
+                font-family: monospace !important;
+            }
+            input {
+                font-family: monospace !important;
+            }
+        </style>
+        """,
+    unsafe_allow_html=True,
+)
+
+st.markdown(
+    """
+        # I Bet You Did Not Mean That
+
+        Official HF Space for the paper [*I Bet You Did Not Mean That: Testing Semantci Importance via Betting*](https://arxiv.org/pdf/2405.19146), by [Jacopo Teneggi](https://jacopoteneggi.github.io) and [Jeremias Sulam](https://sites.google.com/view/jsulam).
+
+        ---
+        """,
+)
+
+
+def load_clip():
+    model, _, preprocess = open_clip.create_model_and_transforms(
+        "hf-hub:laion/CLIP-ViT-B-32-laion2B-s34B-b79K"
+    )
+    tokenizer = open_clip.get_tokenizer("hf-hub:laion/CLIP-ViT-B-32-laion2B-s34B-b79K")
+
+
+def test(
+    image, class_name, concepts, cardinality, model_name, dataset_name="imagenette"
+):
+    print("test!")
+
+
+if __name__ == "__main__":
+    main()

app_lib/main.py

@@ -0,0 +1,51 @@
+import streamlit as st
+
+from app_lib.user_input import (
+    get_cardinality,
+    get_class_name,
+    get_concepts,
+    get_image,
+    get_model_name,
+)
+
+
+def main():
+    columns = st.columns([0.40, 0.60])
+
+    with columns[0]:
+        model_name = get_model_name()
+
+        row1 = st.columns(2)
+        row2 = st.columns(2)
+
+        with row1[0]:
+            image = get_image()
+            st.image(image, use_column_width=True)
+        with row1[1]:
+            class_name, class_ready, class_error = get_class_name()
+            concepts, concepts_ready, concepts_error = get_concepts()
+            cardinality = get_cardinality(concepts, concepts_ready)
+
+        with row2[0]:
+            change_image_button = st.button("Change Image", use_container_width=True)
+            if change_image_button:
+                st.session_state.sidebar_state = "expanded"
+                st.experimental_rerun()
+        with row2[1]:
+            ready = class_ready and concepts_ready
+
+            error_message = ""
+            if class_error is not None:
+                error_message += f"- {class_error}\n"
+            if concepts_error is not None:
+                error_message += f"- {concepts_error}\n"
+
+            test_button = st.button(
+                "Test",
+                help=None if ready else error_message,
+                use_container_width=True,
+                disabled=not ready,
+            )
+
+            if test_button:
+                test(image, class_name, concepts, cardinality, model_name)

app_lib/user_input.py

@@ -0,0 +1,83 @@
+import streamlit as st
+from PIL import Image
+from streamlit_image_select import image_select
+
+from app_lib.utils import SUPPORTED_MODELS
+
+
+def _validate_class_name(class_name):
+    if class_name is None:
+        return (False, "Class name cannot be empty.")
+    if class_name.strip() == "":
+        return (False, "Class name cannot be empty.")
+    return (True, None)
+
+
+def _validate_concepts(concepts):
+    if len(concepts) < 3:
+        return (False, "You must provide at least 3 concepts")
+    if len(concepts) > 10:
+        return (False, "Maximum 10 concepts allowed")
+    return (True, None)
+
+
+def get_model_name():
+    return st.selectbox(
+        "Choose a model to test",
+        options=SUPPORTED_MODELS,
+        help="Name of the vision-language model to test the predictions of.",
+    )
+
+
+def get_image():
+    with st.sidebar:
+        uploaded_file = st.file_uploader("Upload an image", type=["jpg", "png", "jpeg"])
+        image = uploaded_file or image_select(
+            label="or select one",
+            images=[
+                "assets/ace.jpg",
+                "assets/ace.jpg",
+                "assets/ace.jpg",
+                "assets/ace.jpg",
+            ],
+        )
+    return Image.open(image)
+
+
+def get_class_name():
+    class_name = st.text_input(
+        "Class to test",
+        help="Name of the class to build the zero-shot CLIP classifier with.",
+        value="cat",
+    )
+
+    class_ready, class_error = _validate_class_name(class_name)
+    return class_name, class_ready, class_error
+
+
+def get_concepts():
+    concepts = st.text_area(
+        "Concepts to test (max 10)",
+        help="List of concepts to test the predictions of the model with. Write one concept per line.",
+        height=160,
+        value="piano\ncute\nwhiskers\nmusic\nwild",
+    )
+    concepts = concepts.split("\n")
+    concepts = [concept.strip() for concept in concepts]
+    concepts = [concept for concept in concepts if concept != ""]
+    concepts = list(set(concepts))
+
+    concepts_ready, concepts_error = _validate_concepts(concepts)
+    return concepts, concepts_ready, concepts_error
+
+
+def get_cardinality(concepts, concepts_ready):
+    return st.slider(
+        "Size of conditioning set",
+        help="The number of concepts to condition model predictions on.",
+        min_value=1,
+        max_value=max(2, len(concepts) - 1),
+        value=1,
+        step=1,
+        disabled=not concepts_ready,
+    )

app_lib/utils.py

@@ -0,0 +1,14 @@
+from huggingface_hub import hf_hub_download
+
+supported_models_path = hf_hub_download(
+    repo_id="jacopoteneggi/IBYDMT",
+    filename="supported_models.txt",
+    repo_type="dataset",
+)
+
+SUPPORTED_MODELS = []
+with open(supported_models_path, "r") as f:
+    for line in f:
+        line = line.strip()
+        model_name, _ = line.split(",")
+        SUPPORTED_MODELS.append(model_name)

requirements.txt

@@ -0,0 +1,3 @@
+streamlit-image-select
+clip @ git+https://github.com/openai/CLIP@main
+open_clip_torch