will not load due to hardcoded paths

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,9 +1,233 @@
+import os
+from pathlib import Path
+import pandas as pd
 import gradio as gr
+from collections import OrderedDict
+from PIL import Image, ImageDraw, ImageFont
+from io import BytesIO
+import PyPDF2
+import pdf2image
 
+MAX_PAGES = 50
+MAX_PDF_SIZE = 100000000  # almost 100MB
+MIN_WIDTH, MIN_HEIGHT = 150, 150
 
-def greet(name):
-    return "Hello " + name + "!!"
+"""
+Load diagnostic dataset
 
+Have pointer to local PDF/grid files
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-iface.launch()
+Visualize PDF/grid files based on slider values and (randonly) sampled combination of sliders
+
+--> truly interactive visualization of diagnostic samples and their questions
+
+"""
+
+PDF_PATH = Path("/home/jordy/Downloads/DUDE_train-val-test_binaries/PDF")
+DIAGNOSTIC_PATH = "/home/jordy/code/DUchallenge/DUeval/diagnostic_test-updated.csv"  # need access to local path; otherwise will not work
+
+answer_types = {
+    "abstractive": "Abstractive",
+    "extractive": "Extractive",
+    "not-answerable": "Not Answerable",
+    "list/abstractive": "Abstractive List",
+    "list/extractive": "Extractive List",
+}
+
+DIAGNOSTIC_TEST = None
+if os.path.exists(DIAGNOSTIC_PATH):
+    DIAGNOSTIC_TEST = pd.read_csv(DIAGNOSTIC_PATH)
+
+    meta_cats = OrderedDict(
+        {
+            "complexity": ["meta", "multihop", "other_hard", "simple", None],
+            "evidence": [
+                "handwriting",
+                "layout",
+                "plain",
+                "table_or_list",
+                "visual_chart",
+                "visual_checkbox",
+                "visual_color",
+                "visual_image",
+                "visual_logo",
+                "visual_map",
+                "visual_other",
+                "visual_signature",
+                "visual_stamp",
+                None,
+            ],
+            "form": ["date", "numeric", "other", "proper", None],
+            "operation": ["arithmetic", "comparison", "counting", "normalization", None],
+            "type": ["abstractive", "extractive", None],
+        }
+    )
+    diagnostic_cats = [
+        "complexity_meta",
+        "complexity_multihop",
+        "complexity_other_hard",
+        "complexity_simple",
+        "evidence_handwriting",
+        "evidence_layout",
+        "evidence_plain",
+        "evidence_table_or_list",
+        "evidence_visual_chart",
+        "evidence_visual_checkbox",
+        "evidence_visual_color",
+        "evidence_visual_image",
+        "evidence_visual_logo",
+        "evidence_visual_map",
+        "evidence_visual_other",
+        "evidence_visual_signature",
+        "evidence_visual_stamp",
+        "form_date",
+        "form_numeric",
+        "form_other",
+        "form_proper",
+        "operation_arithmetic",
+        "operation_comparison",
+        "operation_counting",
+        "operation_normalization",
+        "type_abstractive",
+        "type_extractive",
+        "num_pages",
+        "num_tokens",
+    ]
+    # DIAGNOSTIC_TEST = DIAGNOSTIC_TEST[interest_cols + ["row_hash"]]
+
+sliders = [gr.Dropdown(choices=choices, value=choices[-1], label=label) for label, choices in meta_cats.items()]
+
+slider_defaults = [None, "visual_checkbox", None, None, None]  # [slider.value for slider in sliders]
+
+
+def equal_image_grid(images):
+    def compute_grid(n, max_cols=6):
+        equalDivisor = int(n**0.5)
+        cols = min(equalDivisor, max_cols)
+        rows = equalDivisor
+        if rows * cols >= n:
+            return rows, cols
+        cols += 1
+        if rows * cols >= n:
+            return rows, cols
+        while rows * cols < n:
+            rows += 1
+        return rows, cols
+
+    # assert len(images) == rows*cols
+    rows, cols = compute_grid(len(images))
+
+    # rescaling to min width [height padding]
+    images = [im for im in images if (im.height > 0) and (im.width > 0)]  # could be NA
+
+    min_width = min(im.width for im in images)
+    images = [im.resize((min_width, int(im.height * min_width / im.width)), resample=Image.BICUBIC) for im in images]
+
+    w, h = max([img.size[0] for img in images]), max([img.size[1] for img in images])
+
+    grid = Image.new("RGB", size=(cols * w, rows * h))
+    grid_w, grid_h = grid.size
+
+    for i, img in enumerate(images):
+        grid.paste(img, box=(i % cols * w, i // cols * h))
+    return grid
+
+
+def add_pagenumbers(im_list, height_scale=40):
+    def add_pagenumber(image, i):
+        width, height = image.size
+        draw = ImageDraw.Draw(image)
+        fontsize = int((width * height) ** (0.5) / height_scale)
+        font = ImageFont.truetype("Arial.ttf", fontsize)
+        margin = int(2 * fontsize)
+        draw.text(
+            (width - margin, height - margin),
+            str(i + 1),
+            fill="#D00917",
+            font=font,
+            spacing=4,
+            align="right",
+        )
+
+    for i, image in enumerate(im_list):
+        add_pagenumber(image, i)
+
+
+def pdf_to_grid(pdf_path):
+
+
+    reader = PyPDF2.PdfReader(pdf_path)
+    reached_page_limit = False
+    images = []
+    try:
+        for p, page in enumerate(reader.pages):
+            if reached_page_limit:
+                break
+            for image in page.images:
+                im = Image.open(BytesIO(image.data))
+                if im.width < MIN_WIDTH and im.height < MIN_HEIGHT:
+                    continue
+                images.append(im)
+    except Exception as e:
+        print(f"{pdf_path} PyPDF get_images {e}")
+        images = pdf2image.convert_from_path(pdf_path)
+
+    # simpler but slower
+    # images = pdf2image.convert_from_path(pdf_path)
+
+    if len(images) == 0:
+        return None
+    add_pagenumbers(images)
+    return equal_image_grid(images)
+
+
+def main(complexity, evidence, form, operation, type):
+    # need to write a query on diagnostic test and sample from it based on slider values
+    # then return the sample
+    query = " and ".join(
+        [
+            f"{cat}_{val} == {True}"
+            for cat, val in zip(meta_cats.keys(), [complexity, evidence, form, operation, type])
+            if val
+        ]
+    )
+    results = DIAGNOSTIC_TEST.query(query)
+    if len(results) == 0:
+        return f"No results found for query {query}", "", "", ""
+
+    for i, sample in results.sample(frac=1).iterrows():
+        print("Sampled: ", sample)
+
+        # first get PDF file
+        PDF, grid = None, None
+        pdf_path = PDF_PATH / "test" / (sample["nhash"] + ".pdf")
+        if not os.path.exists(pdf_path):
+            continue
+        PDF = pdf_path
+        grid = pdf_to_grid(pdf_path)
+        if not grid:
+            continue
+        # opem and visualize as grid image
+
+        question, answer = sample["question"], sample["answer"]
+
+        # get columns where sample is True
+        diagnostics = ", ".join([cat for cat in diagnostic_cats if sample[cat]])
+
+        return question, answer, diagnostics, grid, PDF
+
+
+# test
+q, a, d, im, f = main(*slider_defaults)
+
+
+outputs = [
+    gr.Textbox(label="question"),
+    gr.Textbox(label="answer"),
+    gr.Textbox(label="diagnostics"),
+    gr.Image(label="image grid of PDF"),
+    gr.File(label="PDF"),
+]
+
+iface = gr.Interface(fn=main, inputs=sliders, outputs=outputs, description="Visualize diagnostic samples from DUDE")
+iface.launch(share=False)