add SPICE dependency jar

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jar filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,221 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+
+# Custom additions
+lib/stanford*.jar
+!lib/
+**/.DS_Store

.spice.py

@@ -0,0 +1,95 @@
+import os
+import subprocess
+import json
+import numpy as np
+import tempfile
+
+# Assumes spice.jar is in the same directory as spice.py.  Change as needed.
+SPICE_JAR = "spice-1.0.jar"
+TEMP_DIR = "tmp"
+CACHE_DIR = "cache"
+
+
+class Spice:
+    """
+    Main Class to compute the SPICE metric
+    """
+
+    def float_convert(self, obj):
+        try:
+            return float(obj)
+        except (ValueError, TypeError):
+            return np.nan
+
+    def compute_score(self, gts, res):
+        assert sorted(gts.keys()) == sorted(res.keys())
+        imgIds = sorted(gts.keys())
+
+        # Prepare temp input file for the SPICE scorer
+        input_data = []
+        for id in imgIds:
+            hypo = res[id]
+            ref = gts[id]
+
+            # Sanity check.
+            assert type(hypo) is list
+            assert len(hypo) == 1
+            assert type(ref) is list
+            assert len(ref) >= 1
+
+            input_data.append({"image_id": id, "test": hypo[0], "refs": ref})
+
+        cwd = os.path.dirname(os.path.abspath(__file__))
+        temp_dir = os.path.join(cwd, TEMP_DIR)
+        if not os.path.exists(temp_dir):
+            os.makedirs(temp_dir)
+        in_file = tempfile.NamedTemporaryFile(delete=False, dir=temp_dir)
+        json.dump(input_data, in_file, indent=2)
+        in_file.close()
+
+        # Start job
+        out_file = tempfile.NamedTemporaryFile(delete=False, dir=temp_dir)
+        out_file.close()
+        cache_dir = os.path.join(cwd, CACHE_DIR)
+        if not os.path.exists(cache_dir):
+            os.makedirs(cache_dir)
+        spice_cmd = [
+            "java",
+            "-jar",
+            "-Xmx8G",
+            SPICE_JAR,
+            in_file.name,
+            "-cache",
+            cache_dir,
+            "-out",
+            out_file.name,
+            "-subset",
+            "-silent",
+        ]
+        subprocess.check_call(spice_cmd, cwd=os.path.dirname(os.path.abspath(__file__)))
+
+        # Read and process results
+        with open(out_file.name) as data_file:
+            results = json.load(data_file)
+        os.remove(in_file.name)
+        os.remove(out_file.name)
+
+        imgId_to_scores = {}
+        spice_scores = []
+        for item in results:
+            imgId_to_scores[item["image_id"]] = item["scores"]
+            spice_scores.append(self.float_convert(item["scores"]["All"]["f"]))
+        average_score = np.mean(np.array(spice_scores))
+        scores = []
+        for image_id in imgIds:
+            # Convert none to NaN before saving scores over subcategories
+            score_set = {}
+            for category, score_tuple in imgId_to_scores[image_id].iteritems():
+                score_set[category] = {
+                    k: self.float_convert(v) for k, v in score_tuple.items()
+                }
+            scores.append(score_set)
+        return average_score, scores
+
+    def method(self):
+        return "SPICE"

get_stanford_models.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env sh
+# This script downloads the Stanford CoreNLP models.
+
+CORENLP=stanford-corenlp-full-2015-12-09
+SPICELIB=lib
+
+DIR="$( cd "$(dirname "$0")" ; pwd -P )"
+cd $DIR
+
+echo "Downloading..."
+
+wget http://nlp.stanford.edu/software/$CORENLP.zip
+
+echo "Unzipping..."
+
+mkdir -p .tmp
+unzip $CORENLP.zip -d .tmp/
+mv .tmp/$CORENLP/stanford-corenlp-3.6.0.jar $SPICELIB/
+mv .tmp/$CORENLP/stanford-corenlp-3.6.0-models.jar $SPICELIB/
+rm -f stanford-corenlp-full-2015-12-09.zip
+rm -rf .tmp
+
+echo "Done."

lib/Meteor-1.5.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:014a3627d277a6f4ba5b44a20733f8dff28bbb428204c36abfda56565658ba09
+size 6318623

lib/SceneGraphParser-1.0.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a1b7c3f896dd135f9539ba41d65b155827dfc312893bfd5bb835c9f9ae310a
+size 164039

lib/ejml-0.23.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12d44e81fdd661ad2aac6384511facf3d8fa003c3ce8d9cec5f22827d1819a26
+size 301506

lib/fst-2.47.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c871febc859cf72dbed1c6c1eebc5600c3d93c3e986feccf4623cc5086098cc
+size 380882

lib/guava-19.0.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58d4cc2e05ebb012bbac568b032f75623be1cb6fb096f3c60c72a86f7f057de4
+size 2308517

lib/hamcrest-core-1.3.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66fdef91e9739348df7a096aa384a5685f4e875584cce89386a7a47251c4d8e9
+size 45024

lib/jackson-core-2.5.3.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9a900c30c702d371be2d3714938dfa6de616599c8eb39f8e3cf7aedd62dfd32
+size 229998

lib/javassist-3.19.0-GA.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d19c1ef43ccd9cb1b39466bb2f1c8e45c2b6752f1e13a3dfb60096543d1791fa
+size 749499

lib/json-simple-1.1.1.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e69696892b88b41c55d49ab2fdcc21eead92bf54acc588c0050596c3b75199c
+size 23931

lib/junit-4.12.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59721f0805e223d84b90677887d9ff567dc534d7c502ca903c0c2b17f05c116a
+size 314932

lib/lmdbjni-0.4.6.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4e8c51eb4ef05abe57d2187e76f5c335e4ef16d8503329eb7dbc5ae7a90529c
+size 85980

lib/lmdbjni-linux64-0.4.6.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b7773db873aaeec399a50113c0a4e121df0eff5e724607f7bc8171452107d94
+size 385952

lib/lmdbjni-osx64-0.4.6.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d66778037b5689e7c6daf22a7140722f2085da4ae479bceda4a4be487d55460
+size 106300

lib/lmdbjni-win64-0.4.6.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:969c6aa59893b82856a6832da640672b0f23f1d72f70d7b04296c71970d75b79
+size 72671

lib/objenesis-2.4.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:090416e160a7e398f79e0149a7b05ef328b1843898a34e08c1cc170128d326d0
+size 51287

lib/slf4j-api-1.7.12.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0aee9a77a4940d72932b0d0d9557793f872e66a03f598e473f45e7efecdccf99
+size 32127

lib/slf4j-simple-1.7.21.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a20179cc3ded2d7e2f5def4ccb34b2d0836e2f8c9e79352a6babd6a6559cc8e0
+size 10905

spice-1.0.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c6a73ae6dd108667047c1818707590413934dcb9f1a62cef6469059b41b3d59
+size 19751099