initial commit

.gitattributes

@@ -0,0 +1,2 @@
+*.tsv filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,64 @@
+---
+title: ExtEnD
+emoji: 🚀
+colorFrom: green
+colorTo: gray
+sdk: streamlit
+sdk_version: 1.5.0
+app_file: app.py
+pinned: false
+license: cc-by-nc-sa-4.0
+---
+
+ <h1 align ="center"> ExtEnD: Extractive Entity Disambiguation </h1>
+
+<p align="center">
+    <a href="https://sunglasses-ai.github.io/classy/">
+        <img alt="Python" src="https://img.shields.io/badge/-classy%200.2.1-black?style=for-the-badge&logoColor=white&logo=data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4NCjwhLS0gR2VuZXJhdG9yOiBBZG9iZSBJbGx1c3RyYXRvciAxNy4wLjAsIFNWRyBFeHBvcnQgUGx1Zy1JbiAuIFNWRyBWZXJzaW9uOiA2LjAwIEJ1aWxkIDApICAtLT4NCjwhRE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cudzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+DQo8c3ZnIHZlcnNpb249IjEuMSIgaWQ9IkxpdmVsbG9fMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgeD0iMHB4IiB5PSIwcHgiDQoJIHdpZHRoPSI0MzcuNnB4IiBoZWlnaHQ9IjQxMy45NzhweCIgdmlld0JveD0iMCAwIDQzNy42IDQxMy45NzgiIGVuYWJsZS1iYWNrZ3JvdW5kPSJuZXcgMCAwIDQzNy42IDQxMy45NzgiIHhtbDpzcGFjZT0icHJlc2VydmUiPg0KPGc+DQoJPHBhdGggZmlsbD0iI0ZGQ0MwMCIgZD0iTTM5NC42NjgsMTczLjAzOGMtMS4wMTgsMC0yLjAyLDAuMDY1LTMuMDE1LDAuMTUyQzM3NS42NjUsODIuODExLDI5Ni43OSwxNC4xNDYsMjAxLjgyMywxNC4xNDYNCgkJQzk1LjMxOSwxNC4xNDYsOC45OCwxMDAuNDg1LDguOTgsMjA2Ljk4OWMwLDEwNi41MDUsODYuMzM5LDE5Mi44NDMsMTkyLjg0MywxOTIuODQzYzk0Ljk2NywwLDE3My44NDItNjguNjY1LDE4OS44MjktMTU5LjA0NA0KCQljMC45OTUsMC4wODcsMS45OTcsMC4xNTIsMy4wMTUsMC4xNTJjMTguNzUxLDAsMzMuOTUyLTE1LjIsMzMuOTUyLTMzLjk1MkM0MjguNjIsMTg4LjIzOSw0MTMuNDE5LDE3My4wMzgsMzk0LjY2OCwxNzMuMDM4eg0KCQkgTTIwMS44MjMsMzQ2Ljg2OWMtNzcuMTMsMC0xMzkuODgtNjIuNzUtMTM5Ljg4LTEzOS44OGMwLTc3LjEyOSw2Mi43NS0xMzkuODc5LDEzOS44OC0xMzkuODc5czEzOS44OCw2Mi43NSwxMzkuODgsMTM5Ljg3OQ0KCQlDMzQxLjcwMywyODQuMTE5LDI3OC45NTMsMzQ2Ljg2OSwyMDEuODIzLDM0Ni44Njl6Ii8+DQoJPGc+DQoJCTxwYXRoIGZpbGw9IiNGRkNDMDAiIGQ9Ik0xMTQuOTA3LDIzMy40NzNjLTE0LjYyNiwwLTI2LjQ4My0xMS44NTYtMjYuNDgzLTI2LjQ4M2MwLTYyLjUyOCw1MC44NzEtMTEzLjQwMiwxMTMuMzk4LTExMy40MDINCgkJCWMxNC42MjYsMCwyNi40ODMsMTEuODU2LDI2LjQ4MywyNi40ODNzLTExLjg1NiwyNi40ODMtMjYuNDgzLDI2LjQ4M2MtMzMuMzI0LDAtNjAuNDMzLDI3LjExMi02MC40MzMsNjAuNDM2DQoJCQlDMTQxLjM5LDIyMS42MTcsMTI5LjUzNCwyMzMuNDczLDExNC45MDcsMjMzLjQ3M3oiLz4NCgk8L2c+DQo8L2c+DQo8L3N2Zz4NCg==">
+    </a>
+    <a href="">
+        <img alt="Python" src="https://img.shields.io/badge/Python 3.8--3.9-blue?style=for-the-badge&logo=python&logoColor=white">
+    </a>    
+    <a href="https://pytorch.org/get-started/locally/">
+        <img alt="PyTorch" src="https://img.shields.io/badge/PyTorch 1.9-ee4c2c?style=for-the-badge&logo=pytorch&logoColor=white">
+    </a>
+    <a href="https://spacy.io/">
+        <img alt="plugin: spacy" src="https://img.shields.io/badge/plugin%20for-spaCy%203.2-09A3D5.svg?style=for-the-badge&labelColor=gray">
+    </a>
+    <a href="https://black.readthedocs.io/en/stable/">
+        <img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-black.svg?style=for-the-badge&labelColor=gray">
+    </a>
+        
+</p>
+
+This space contains the demo of [ExtEnD: Extractive Entity Disambiguation](https://www.researchgate.net/publication/359392427_ExtEnD_Extractive_Entity_Disambiguation), 
+a novel approach to Entity Disambiguation (i.e. the task of linking a mention in context with its most suitable entity in a reference knowledge base) where we reformulate
+this task as a text extraction problem. This work was accepted at ACL 2022.
+
+If you find this demo, our paper, code or framework useful, please reference this work in your paper:
+
+```
+@inproceedings{barba-etal-2021-consec,
+    title = "{E}xt{E}n{D}: Extractive Entity Disambiguation",
+    author = "Barba, Edoardo  and
+      Procopio, Luigi  and
+      Navigli, Roberto",
+    booktitle = "Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics",
+    month = may,
+    year = "2022",
+    address = "Online and Dublin, Ireland",
+    publisher = "Association for Computational Linguistics",
+}
+```
+
+![ExtEnD Image](data/repo-assets/extend_formulation.png)
+
+## Acknowledgments
+
+The authors gratefully acknowledge the support of the ERC Consolidator Grant MOUSSE No. 726487 under the European Union’s Horizon 2020 research and innovation programme.
+
+This work was supported in part by the MIUR under grant “Dipartimenti di eccellenza 2018-2022” of the Department of Computer Science of the Sapienza University of Rome.
+
+## License
+
+This work is under the [Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0) license](https://creativecommons.org/licenses/by-nc-sa/4.0/).

app.py

@@ -0,0 +1,176 @@
+import argparse
+import html
+import time
+
+from extend import spacy_component  # this is needed to register the spacy component
+
+import spacy
+import streamlit as st
+from annotated_text import annotation
+from classy.scripts.model.demo import tabbed_navigation
+from classy.utils.streamlit import get_md_200_random_color_generator
+
+
+def main(
+        model_checkpoint_path: str,
+        default_inventory_path: str,
+        cuda_device: int,
+):
+    # setup examples
+    examples = [
+        "Italy beat England and won Euro 2021.",
+        "Japan began the defence of their Asian Cup title with a lucky 2-1 win against Syria in a Group C championship match on Friday.",
+        "The project was coded in Java.",
+    ]
+
+    # css rules
+    st.write(
+        """
+            <style type="text/css">
+                a {
+                    text-decoration: none !important;
+                }
+            </style>
+        """,
+        unsafe_allow_html=True,
+    )
+
+    # setup header
+    st.markdown("<h1 style='text-align: center;'>ExtEnD: Extractive Entity Disambiguation</h1>", unsafe_allow_html=True)
+    st.write(
+        """
+            <div align="center">
+                <a href="https://sunglasses-ai.github.io/classy/">
+                    <img alt="Python" style="height: 3em; margin: 0 1em" src="data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4NCjwhLS0gR2VuZXJhdG9yOiBBZG9iZSBJbGx1c3RyYXRvciAxNy4wLjAsIFNWRyBFeHBvcnQgUGx1Zy1JbiAuIFNWRyBWZXJzaW9uOiA2LjAwIEJ1aWxkIDApICAtLT4NCjwhRE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cudzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+DQo8c3ZnIHZlcnNpb249IjEuMSIgaWQ9IkxpdmVsbG9fMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgeD0iMHB4IiB5PSIwcHgiDQoJIHdpZHRoPSI0MzcuNnB4IiBoZWlnaHQ9IjQxMy45NzhweCIgdmlld0JveD0iMCAwIDQzNy42IDQxMy45NzgiIGVuYWJsZS1iYWNrZ3JvdW5kPSJuZXcgMCAwIDQzNy42IDQxMy45NzgiIHhtbDpzcGFjZT0icHJlc2VydmUiPg0KPGc+DQoJPHBhdGggZmlsbD0iI0ZGQ0MwMCIgZD0iTTM5NC42NjgsMTczLjAzOGMtMS4wMTgsMC0yLjAyLDAuMDY1LTMuMDE1LDAuMTUyQzM3NS42NjUsODIuODExLDI5Ni43OSwxNC4xNDYsMjAxLjgyMywxNC4xNDYNCgkJQzk1LjMxOSwxNC4xNDYsOC45OCwxMDAuNDg1LDguOTgsMjA2Ljk4OWMwLDEwNi41MDUsODYuMzM5LDE5Mi44NDMsMTkyLjg0MywxOTIuODQzYzk0Ljk2NywwLDE3My44NDItNjguNjY1LDE4OS44MjktMTU5LjA0NA0KCQljMC45OTUsMC4wODcsMS45OTcsMC4xNTIsMy4wMTUsMC4xNTJjMTguNzUxLDAsMzMuOTUyLTE1LjIsMzMuOTUyLTMzLjk1MkM0MjguNjIsMTg4LjIzOSw0MTMuNDE5LDE3My4wMzgsMzk0LjY2OCwxNzMuMDM4eg0KCQkgTTIwMS44MjMsMzQ2Ljg2OWMtNzcuMTMsMC0xMzkuODgtNjIuNzUtMTM5Ljg4LTEzOS44OGMwLTc3LjEyOSw2Mi43NS0xMzkuODc5LDEzOS44OC0xMzkuODc5czEzOS44OCw2Mi43NSwxMzkuODgsMTM5Ljg3OQ0KCQlDMzQxLjcwMywyODQuMTE5LDI3OC45NTMsMzQ2Ljg2OSwyMDEuODIzLDM0Ni44Njl6Ii8+DQoJPGc+DQoJCTxwYXRoIGZpbGw9IiNGRkNDMDAiIGQ9Ik0xMTQuOTA3LDIzMy40NzNjLTE0LjYyNiwwLTI2LjQ4My0xMS44NTYtMjYuNDgzLTI2LjQ4M2MwLTYyLjUyOCw1MC44NzEtMTEzLjQwMiwxMTMuMzk4LTExMy40MDINCgkJCWMxNC42MjYsMCwyNi40ODMsMTEuODU2LDI2LjQ4MywyNi40ODNzLTExLjg1NiwyNi40ODMtMjYuNDgzLDI2LjQ4M2MtMzMuMzI0LDAtNjAuNDMzLDI3LjExMi02MC40MzMsNjAuNDM2DQoJCQlDMTQxLjM5LDIyMS42MTcsMTI5LjUzNCwyMzMuNDczLDExNC45MDcsMjMzLjQ3M3oiLz4NCgk8L2c+DQo8L2c+DQo8L3N2Zz4NCg==">
+                </a>
+                <a href="https://spacy.io/" tyle="text-decoration: none">
+                    <img alt="spaCy" style="height: 3em; margin: 0 1em;" src="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgOTAwIDUwMCAxNzUiIHdpZHRoPSIxNTAiIGhlaWdodD0iNTMiPjxwYXRoIGZpbGw9IiMwOUEzRDUiIGQ9Ik02NC44IDk3MC42Yy0xMS4zLTEuMy0xMi4yLTE2LjUtMjYuNy0xNS4yLTcgMC0xMy42IDIuOS0xMy42IDkuNCAwIDkuNyAxNSAxMC42IDI0LjEgMTMuMSAxNS40IDQuNyAzMC40IDcuOSAzMC40IDI0LjcgMCAyMS4zLTE2LjcgMjguNy0zOC43IDI4LjctMTguNCAwLTM3LjEtNi41LTM3LjEtMjMuNSAwLTQuNyA0LjUtOC40IDguOS04LjQgNS41IDAgNy41IDIuMyA5LjQgNi4yIDQuMyA3LjUgOS4xIDExLjYgMjEgMTEuNiA3LjUgMCAxNS4zLTIuOSAxNS4zLTkuNCAwLTkuMy05LjUtMTEuMy0xOS4zLTEzLjYtMTcuNC00LjktMzIuMy03LjQtMzQtMjYuNy0xLjgtMzIuOSA2Ni43LTM0LjEgNzAuNi01LjMtLjMgNS4yLTUuMiA4LjQtMTAuMyA4LjR6bTgxLjUtMjguOGMyNC4xIDAgMzcuNyAyMC4xIDM3LjcgNDQuOSAwIDI0LjktMTMuMiA0NC45LTM3LjcgNDQuOS0xMy42IDAtMjIuMS01LjgtMjguMi0xNC43djMyLjljMCA5LjktMy4yIDE0LjctMTAuNCAxNC43LTguOCAwLTEwLjQtNS42LTEwLjQtMTQuN3YtOTUuNmMwLTcuOCAzLjMtMTIuNiAxMC40LTEyLjYgNi43IDAgMTAuNCA1LjMgMTAuNCAxMi42djIuN2M2LjgtOC41IDE0LjYtMTUuMSAyOC4yLTE1LjF6bS01LjcgNzIuOGMxNC4xIDAgMjAuNC0xMyAyMC40LTI4LjIgMC0xNC44LTYuNC0yOC4yLTIwLjQtMjguMi0xNC43IDAtMjEuNSAxMi4xLTIxLjUgMjguMi4xIDE1LjcgNi45IDI4LjIgMjEuNSAyOC4yem01OS44LTQ5LjNjMC0xNy4zIDE5LjktMjMuNSAzOS4yLTIzLjUgMjcuMSAwIDM4LjIgNy45IDM4LjIgMzR2MjUuMmMwIDYgMy43IDE3LjkgMy43IDIxLjUgMCA1LjUtNSA4LjktMTAuNCA4LjktNiAwLTEwLjQtNy0xMy42LTEyLjEtOC44IDctMTguMSAxMi4xLTMyLjQgMTIuMS0xNS44IDAtMjguMi05LjMtMjguMi0yNC43IDAtMTMuNiA5LjctMjEuNCAyMS41LTI0LjEgMCAuMSAzNy43LTguOSAzNy43LTkgMC0xMS42LTQuMS0xNi43LTE2LjMtMTYuNy0xMC43IDAtMTYuMiAyLjktMjAuNCA5LjQtMy40IDQuOS0yLjkgNy44LTkuNCA3LjgtNS4xIDAtOS42LTMuNi05LjYtOC44em0zMi4yIDUxLjljMTYuNSAwIDIzLjUtOC43IDIzLjUtMjYuMXYtMy43Yy00LjQgMS41LTIyLjQgNi0yNy4zIDYuNy01LjIgMS0xMC40IDQuOS0xMC40IDExIC4yIDYuNyA3LjEgMTIuMSAxNC4yIDEyLjF6TTM1NCA5MDljMjMuMyAwIDQ4LjYgMTMuOSA0OC42IDM2LjEgMCA1LjctNC4zIDEwLjQtOS45IDEwLjQtNy42IDAtOC43LTQuMS0xMi4xLTkuOS01LjYtMTAuMy0xMi4yLTE3LjItMjYuNy0xNy4yLTIyLjMtLjItMzIuMyAxOS0zMi4zIDQyLjggMCAyNCA4LjMgNDEuMyAzMS40IDQxLjMgMTUuMyAwIDIzLjgtOC45IDI4LjItMjAuNCAxLjgtNS4zIDQuOS0xMC40IDExLjYtMTAuNCA1LjIgMCAxMC40IDUuMyAxMC40IDExIDAgMjMuNS0yNCAzOS43LTQ4LjYgMzkuNy0yNyAwLTQyLjMtMTEuNC01MC42LTMwLjQtNC4xLTkuMS02LjctMTguNC02LjctMzEuNC0uNC0zNi40IDIwLjgtNjEuNiA1Ni43LTYxLjZ6bTEzMy4zIDMyLjhjNiAwIDkuNCAzLjkgOS40IDkuOSAwIDIuNC0xLjkgNy4zLTIuNyA5LjlsLTI4LjcgNzUuNGMtNi40IDE2LjQtMTEuMiAyNy43LTMyLjkgMjcuNy0xMC4zIDAtMTkuMy0uOS0xOS4zLTkuOSAwLTUuMiAzLjktNy44IDkuNC03LjggMSAwIDIuNy41IDMuNy41IDEuNiAwIDIuNy41IDMuNy41IDEwLjkgMCAxMi40LTExLjIgMTYuMy0xOC45bC0yNy43LTY4LjVjLTEuNi0zLjctMi43LTYuMi0yLjctOC40IDAtNiA0LjctMTAuNCAxMS0xMC40IDcgMCA5LjggNS41IDExLjYgMTEuNmwxOC4zIDU0LjMgMTguMy01MC4yYzIuNy03LjggMy0xNS43IDEyLjMtMTUuN3oiIC8+IDwvc3ZnPg==">
+                </a>
+            </div> 
+        """,
+        unsafe_allow_html=True,
+    )
+
+    def model_demo():
+
+        @st.cache(allow_output_mutation=True)
+        def load_resources(inventory_path):
+
+            # load nlp
+            nlp = spacy.load("en_core_web_sm")
+            extend_config = dict(
+                checkpoint_path=model_checkpoint_path,
+                mentions_inventory_path=inventory_path,
+                device=cuda_device,
+                tokens_per_batch=10_000,
+            )
+            nlp.add_pipe("extend", after="ner", config=extend_config)
+
+            # mock call to load resources
+            nlp(examples[0])
+
+            # return
+            return nlp
+
+        # read input
+        placeholder = st.selectbox(
+            "Examples",
+            options=examples,
+            index=0,
+        )
+        input_text = st.text_area("Input text to entity-disambiguate", placeholder)
+
+        # custom inventory
+        uploaded_inventory_path = st.file_uploader(
+            "[Optional] Upload custom inventory (tsv file, mention \\t desc1 \\t desc2 \\t)",
+            accept_multiple_files=False, type=["tsv"])
+        if uploaded_inventory_path is not None:
+            inventory_path = f"data/inventories/{uploaded_inventory_path.name}"
+            with open(inventory_path, "wb") as f:
+                f.write(uploaded_inventory_path.getbuffer())
+        else:
+            inventory_path = default_inventory_path
+
+        if st.button("Classify", key="classify"):
+
+            # load model
+            nlp = load_resources(inventory_path)
+            color_generator = get_md_200_random_color_generator()
+
+            # tag sentence
+            time_start = time.perf_counter()
+            doc = nlp(input_text)
+            time_end = time.perf_counter()
+
+            # extract entities
+            entities = {}
+            for ent in doc.ents:
+                if ent._.disambiguated_entity is not None:
+                    entities[ent.start_char] = (
+                        ent.start_char,
+                        ent.end_char,
+                        ent.text,
+                        ent._.disambiguated_entity,
+                    )
+
+            # create annotated html components
+
+            annotated_html_components = []
+
+            assert all(any(t.idx == _s for t in doc) for _s in entities)
+            it = iter(list(doc))
+            while True:
+                try:
+                    t = next(it)
+                except StopIteration:
+                    break
+                if t.idx in entities:
+                    _start, _end, _text, _entity = entities[t.idx]
+                    while t.idx + len(t) != _end:
+                        t = next(it)
+                    annotated_html_components.append(
+                        str(annotation(*(_text, _entity, color_generator())))
+                    )
+                else:
+                    annotated_html_components.append(str(html.escape(t.text)))
+
+            st.markdown(
+                "\n".join(
+                    [
+                        "<div>",
+                        *annotated_html_components,
+                        "<p></p>"
+                        f'<div style="text-align: right"><p style="color: gray">Time: {(time_end - time_start):.2f}s</p></div>'
+                        "</div>",
+                    ]
+                ),
+                unsafe_allow_html=True,
+            )
+
+    def hiw():
+        st.markdown("ExtEnD frames Entity Disambiguation as a text extraction problem:")
+        st.image("data/repo-assets/extend_formulation.png", caption="ExtEnD Formulation")
+        st.markdown("""            
+            Given the sentence *After a long fight Superman saved Metropolis*, where *Superman* is the mention
+            to disambiguate, ExtEnD first concatenates the descriptions of all the possible candidates of *Superman* in the
+            inventory and then selects the span whose description best suits the mention in its context.
+
+            To convert this task to end2end entity linking, as we do in *Model demo*, we leverage spaCy 
+            (more specifically, its NER) and run ExtEnD on each named entity spaCy identifies 
+            (if the corresponding mention is contained in the inventory).
+        """)
+
+    def abstract():
+        st.write(
+            """
+            Word Sense Disambiguation (WSD) is a historical NLP task aimed at linking words in contexts to discrete sense inventories and it is usually cast as a multi-label classification task. Recently, several neural approaches have employed sense definitions to better represent word meanings. Yet, these approaches do not observe the input sentence and the sense definition candidates all at once, thus potentially reducing the model performance and generalization power. We cope with this issue by reframing WSD as a span extraction problem --- which we called Extractive Sense Comprehension (ESC) --- and propose ESCHER, a transformer-based neural architecture for this new formulation. By means of an extensive array of experiments, we show that ESC unleashes the full potential of our model, leading it to outdo all of its competitors and to set a new state of the art on the English WSD task. In the few-shot scenario, ESCHER proves to exploit training data efficiently, attaining the same performance as its closest competitor while relying on almost three times fewer annotations. Furthermore, ESCHER can nimbly combine data annotated with senses from different lexical resources, achieving performances that were previously out of everyone's reach. The model along with data is available at https://github.com/SapienzaNLP/esc.
+        """
+        )
+
+    tabs = dict(model=("Model demo", model_demo), hiw=("How it works", hiw), abstract=("Abstract", abstract))
+
+    tabbed_navigation(tabs, "model")
+
+
+if __name__ == "__main__":
+    main("experiments/extend-longformer-large/2021-10-22/09-11-39/checkpoints/best.ckpt", "data/inventories/aida.tsv", cuda_device=-1)

data/inventories/aida.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79aee10d058068ddfd791c853658c406e8c81dff6a2ea8fabcc7bb468df81986
+size 34209452

experiments/extend-longformer-large/2021-10-22/09-11-39/.hydra/config.bk.yaml

@@ -0,0 +1,171 @@
+callbacks:
+  callbacks: []
+data:
+  datamodule:
+    _target_: classy.data.data_modules.ClassyDataModule
+    task: ${task}
+    dataset_path: data/aida
+    train_dataset:
+      _target_: classy.data.dataset.hf.classification.HFQADataset.from_file
+      transformer_model: ${transformer_model}
+      additional_special_tokens: ${model.additional_special_tokens}
+      min_length: 5
+      max_length: 500
+      tokens_per_batch: 2000
+      max_batch_size: 10
+      section_size: 10000
+      prebatch: true
+      materialize: false
+      for_inference: false
+    validation_dataset:
+      _target_: classy.data.dataset.hf.classification.HFQADataset.from_file
+      transformer_model: ${transformer_model}
+      additional_special_tokens: ${model.additional_special_tokens}
+      min_length: 5
+      max_length: 500
+      tokens_per_batch: 2000
+      max_batch_size: 10
+      section_size: 10000
+      prebatch: true
+      materialize: true
+      for_inference: true
+    validation_split_size: 0.1
+    test_split_size: 0.1
+    max_nontrain_split_size: 10000
+    shuffle_dataset: true
+device:
+  gpus:
+  - 0
+  precision: 32
+  amp_level: O0
+model:
+  _target_: classy.pl_modules.hf.classification.HFQAPLModule
+  transformer_model: ${transformer_model}
+  additional_special_tokens: []
+  optim_conf:
+    _target_: classy.optim.factories.RAdamFactory
+    lr: 1.0e-05
+    weight_decay: 0.01
+    no_decay_params:
+    - bias
+    - LayerNorm.weight
+prediction:
+  dataset:
+    _target_: classy.data.dataset.hf.classification.HFQADataset.from_samples
+    transformer_model: ${transformer_model}
+    additional_special_tokens: ${model.additional_special_tokens}
+    min_length: -1
+    max_length: -1
+    tokens_per_batch: 800
+    max_batch_size: -1
+    section_size: 10000
+    prebatch: true
+    materialize: false
+    for_inference: true
+training:
+  seed: 12
+  pl_trainer:
+    _target_: pytorch_lightning.Trainer
+    accumulate_grad_batches: 4
+    gradient_clip_val: 10.0
+    val_check_interval: 1.0
+    max_steps: 1000000
+  early_stopping_callback:
+    _target_: pytorch_lightning.callbacks.EarlyStopping
+    monitor: ${callbacks_monitor}
+    mode: ${callbacks_mode}
+    patience: 25
+  model_checkpoint_callback:
+    _target_: classy.pl_callbacks.best_checkpoint.ModelCheckpointWithBest
+    monitor: ${callbacks_monitor}
+    mode: ${callbacks_mode}
+    verbose: true
+    save_top_k: 3
+    dirpath: checkpoints
+    save_last: true
+  resume_from: null
+logging:
+  wandb:
+    use_wandb: true
+    project_name: esc-ed
+    experiment_name: aida-longformer-large-*sep-gam-cand-shuffle
+    anonymous: null
+    run_id: null
+task: qa
+project_name: classy
+exp_name: esc-aida-longformer-large-gam-cand-shuffle
+exp_folder: ./experiments/${exp_name}
+transformer_model: bert-base-cased
+callbacks_monitor: val_accuracy
+callbacks_mode: max
+profiles:
+  supported_tasks:
+  - qa
+  - sentence-pair
+  - sequence
+  - token
+  - generation
+  transformer_model: allenai/longformer-large-4096
+  candidates_separator: '*'
+  training:
+    pl_trainer:
+      accumulate_grad_batches: 8
+      val_check_interval: 2048
+      max_steps: 100000
+  model:
+    _target_: src.esc_ed_module.ESCModule
+    additional_special_tokens: []
+    transformer_model: ${transformer_model}
+    attention_window: 64
+    modify_global_attention: true
+    optim_conf:
+      _target_: classy.optim.factories.RAdamFactory
+      lr: 1.0e-05
+      weight_decay: 0.01
+      no_decay_params:
+      - bias
+      - LayerNorm.weight
+  data:
+    datamodule:
+      train_dataset:
+        _target_: src.data.esc_ed_dataset.ESCEDDataset.from_file
+        transformer_model: ${transformer_model}
+        additional_special_tokens: ${model.additional_special_tokens}
+        candidates_separator: ${candidates_separator}
+        shuffle_candidates_prob: 0.0
+        min_length: 0
+        max_length: 1024
+        tokens_per_batch: 1024
+        max_batch_size: 10
+        section_size: 20000
+        prebatch: true
+        materialize: false
+        for_inference: false
+      validation_dataset:
+        _target_: src.data.esc_ed_dataset.ESCEDDataset.from_file
+        transformer_model: ${transformer_model}
+        additional_special_tokens: ${model.additional_special_tokens}
+        candidates_separator: ${candidates_separator}
+        min_length: 0
+        max_length: 1024
+        tokens_per_batch: 2048
+        max_batch_size: 10
+        section_size: 10000
+        prebatch: true
+        materialize: true
+        for_inference: false
+      shuffle_dataset: true
+  prediction:
+    dataset:
+      _target_: src.data.esc_ed_dataset.ESCEDDataset.from_samples
+      transformer_model: ${transformer_model}
+      additional_special_tokens: ${model.additional_special_tokens}
+      candidates_separator: ${candidates_separator}
+      min_length: -1
+      max_length: -1
+      tokens_per_batch: 2048
+      max_batch_size: -1
+      section_size: 10000
+      prebatch: true
+      materialize: false
+      for_inference: true

experiments/extend-longformer-large/2021-10-22/09-11-39/.hydra/config.yaml

@@ -0,0 +1,115 @@
+callbacks:
+  callbacks: []
+data:
+  datamodule:
+    _target_: classy.data.data_modules.ClassyDataModule
+    task: qa
+    dataset_path: data/aida
+    train_dataset:
+      _target_: extend.data.esc_ed_dataset.ESCEDDataset.from_file
+      transformer_model: allenai/longformer-large-4096
+      additional_special_tokens: []
+      candidates_separator: '*'
+      shuffle_candidates_prob: 0.0
+      min_length: 0
+      max_length: 1024
+      tokens_per_batch: 1024
+      max_batch_size: 10
+      section_size: 20000
+      prebatch: true
+      materialize: false
+      for_inference: false
+    validation_dataset:
+      _target_: extend.data.esc_ed_dataset.ESCEDDataset.from_file
+      transformer_model: allenai/longformer-large-4096
+      additional_special_tokens: []
+      candidates_separator: '*'
+      min_length: 0
+      max_length: 1024
+      tokens_per_batch: 2048
+      max_batch_size: 10
+      section_size: 10000
+      prebatch: true
+      materialize: true
+      for_inference: false
+    validation_split_size: 0.1
+    test_split_size: 0.1
+    max_nontrain_split_size: 10000
+    shuffle_dataset: true
+device:
+  gpus:
+  - 0
+  precision: 32
+  amp_level: O0
+model:
+  _target_: extend.esc_ed_module.ESCModule
+  additional_special_tokens: []
+  transformer_model: allenai/longformer-large-4096
+  attention_window: 64
+  modify_global_attention: 0
+  optim_conf:
+    _target_: classy.optim.factories.RAdamFactory
+    lr: 1.0e-05
+    weight_decay: 0.01
+    no_decay_params:
+    - bias
+    - LayerNorm.weight
+prediction:
+  dataset:
+    _target_: extend.data.esc_ed_dataset.ESCEDDataset.from_samples
+    transformer_model: allenai/longformer-large-4096
+    additional_special_tokens: []
+    candidates_separator: '*'
+    min_length: -1
+    max_length: -1
+    tokens_per_batch: 2048
+    max_batch_size: -1
+    section_size: 10000
+    prebatch: true
+    materialize: false
+    for_inference: true
+training:
+  seed: 12
+  pl_trainer:
+    _target_: pytorch_lightning.Trainer
+    accumulate_grad_batches: 8
+    gradient_clip_val: 10.0
+    val_check_interval: 2048
+    max_steps: 100000
+  early_stopping_callback:
+    _target_: pytorch_lightning.callbacks.EarlyStopping
+    monitor: val_accuracy
+    mode: max
+    patience: 25
+  model_checkpoint_callback:
+    _target_: classy.pl_callbacks.best_checkpoint.ModelCheckpointWithBest
+    monitor: val_accuracy
+    mode: max
+    verbose: true
+    save_top_k: 3
+    dirpath: checkpoints
+    save_last: true
+  resume_from: null
+logging:
+  wandb:
+    use_wandb: true
+    project_name: esc-ed
+    experiment_name: aida-longformer-large-*sep-gam-cand-shuffle
+    anonymous: null
+    run_id: 3v74woaz
+task: qa
+project_name: classy
+exp_name: esc-aida-longformer-large-gam-cand-shuffle
+exp_folder: /root/esc-ed/experiments/esc-aida-longformer-large-gam-cand-shuffle
+transformer_model: allenai/longformer-large-4096
+callbacks_monitor: val_accuracy
+callbacks_mode: max
+supported_tasks:
+- qa
+- sentence-pair
+- sequence
+- token
+- generation
+candidates_separator: '*'
+evaluation:
+  _target_: extend.evaluation.InKBF1

experiments/extend-longformer-large/2021-10-22/09-11-39/.hydra/config_post_trainer_init.yaml

@@ -0,0 +1,113 @@
+callbacks:
+  callbacks: []
+data:
+  datamodule:
+    _target_: classy.data.data_modules.ClassyDataModule
+    task: qa
+    dataset_path: /root/esc-ed/data/aida
+    train_dataset:
+      _target_: src.data.esc_ed_dataset.ESCEDDataset.from_file
+      transformer_model: allenai/longformer-large-4096
+      additional_special_tokens: []
+      candidates_separator: '*'
+      shuffle_candidates_prob: 0.0
+      min_length: 0
+      max_length: 1024
+      tokens_per_batch: 1024
+      max_batch_size: 10
+      section_size: 20000
+      prebatch: true
+      materialize: false
+      for_inference: false
+    validation_dataset:
+      _target_: src.data.esc_ed_dataset.ESCEDDataset.from_file
+      transformer_model: allenai/longformer-large-4096
+      additional_special_tokens: []
+      candidates_separator: '*'
+      min_length: 0
+      max_length: 1024
+      tokens_per_batch: 2048
+      max_batch_size: 10
+      section_size: 10000
+      prebatch: true
+      materialize: true
+      for_inference: false
+    validation_split_size: 0.1
+    test_split_size: 0.1
+    max_nontrain_split_size: 10000
+    shuffle_dataset: true
+device:
+  gpus:
+  - 0
+  precision: 32
+  amp_level: O0
+model:
+  _target_: src.esc_ed_module.ESCModule
+  additional_special_tokens: []
+  transformer_model: allenai/longformer-large-4096
+  attention_window: 64
+  modify_global_attention: true
+  optim_conf:
+    _target_: classy.optim.factories.RAdamFactory
+    lr: 1.0e-05
+    weight_decay: 0.01
+    no_decay_params:
+    - bias
+    - LayerNorm.weight
+prediction:
+  dataset:
+    _target_: src.data.esc_ed_dataset.ESCEDDataset.from_samples
+    transformer_model: allenai/longformer-large-4096
+    additional_special_tokens: []
+    candidates_separator: '*'
+    min_length: -1
+    max_length: -1
+    tokens_per_batch: 2048
+    max_batch_size: -1
+    section_size: 10000
+    prebatch: true
+    materialize: false
+    for_inference: true
+training:
+  seed: 12
+  pl_trainer:
+    _target_: pytorch_lightning.Trainer
+    accumulate_grad_batches: 8
+    gradient_clip_val: 10.0
+    val_check_interval: 2048
+    max_steps: 100000
+  early_stopping_callback:
+    _target_: pytorch_lightning.callbacks.EarlyStopping
+    monitor: val_accuracy
+    mode: max
+    patience: 25
+  model_checkpoint_callback:
+    _target_: classy.pl_callbacks.best_checkpoint.ModelCheckpointWithBest
+    monitor: val_accuracy
+    mode: max
+    verbose: true
+    save_top_k: 3
+    dirpath: checkpoints
+    save_last: true
+  resume_from: null
+logging:
+  wandb:
+    use_wandb: true
+    project_name: esc-ed
+    experiment_name: aida-longformer-large-*sep-gam-cand-shuffle
+    anonymous: null
+    run_id: 3v74woaz
+task: qa
+project_name: classy
+exp_name: esc-aida-longformer-large-gam-cand-shuffle
+exp_folder: /root/esc-ed/experiments/esc-aida-longformer-large-gam-cand-shuffle
+transformer_model: allenai/longformer-large-4096
+callbacks_monitor: val_accuracy
+callbacks_mode: max
+supported_tasks:
+- qa
+- sentence-pair
+- sequence
+- token
+- generation
+candidates_separator: '*'

experiments/extend-longformer-large/2021-10-22/09-11-39/.hydra/hydra.yaml

@@ -0,0 +1,172 @@
+hydra:
+  run:
+    dir: ./experiments/${exp_name}/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '%(asctime)s - %(levelname)s - %(name)s - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.job.name}.log
+    root:
+      level: WARN
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra: []
+    task:
+    - +profiles=aida-longformer-large-gam
+    - device=cuda
+    - device.gpus=[0]
+    - exp_name=esc-aida-longformer-large-gam-cand-shuffle
+    - data.datamodule.dataset_path=data/aida
+    - logging.wandb.use_wandb=True
+    - logging.wandb.project_name=esc-ed
+    - logging.wandb.experiment_name=aida-longformer-large-*sep-gam-cand-shuffle
+    - training.early_stopping_callback.patience=25
+  job:
+    name: train
+    override_dirname: +profiles=aida-longformer-large-gam,data.datamodule.dataset_path=data/aida,device.gpus=[0],device=cuda,exp_name=esc-aida-longformer-large-gam-cand-shuffle,logging.wandb.experiment_name=aida-longformer-large-*sep-gam-cand-shuffle,logging.wandb.project_name=esc-ed,logging.wandb.use_wandb=True,training.early_stopping_callback.patience=25
+    id: ???
+    num: ???
+    config_name: qa
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.1.1
+    cwd: /root/esc-ed
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: classy.scripts.cli
+      schema: pkg
+      provider: main
+    - path: configurations
+      schema: pkg
+      provider: classy-searchpath-plugin
+    - path: /root/esc-ed/configurations
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    choices:
+      profiles: aida-longformer-large-gam
+      logging: default
+      training: default
+      prediction: qa
+      model: qa
+      device: cuda
+      data: qa
+      callbacks: empty
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose:
+  - classy

experiments/extend-longformer-large/2021-10-22/09-11-39/.hydra/overrides.yaml

@@ -0,0 +1,9 @@
+- +profiles=aida-longformer-large-gam
+- device=cuda
+- device.gpus=[0]
+- exp_name=esc-aida-longformer-large-gam-cand-shuffle
+- data.datamodule.dataset_path=data/aida
+- logging.wandb.use_wandb=True
+- logging.wandb.project_name=esc-ed
+- logging.wandb.experiment_name=aida-longformer-large-*sep-gam-cand-shuffle
+- training.early_stopping_callback.patience=25

experiments/extend-longformer-large/2021-10-22/09-11-39/checkpoints/best.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daf5ca2a79a3cdcac3864db7a17b3a9fb084457c7167a7d8e656d8f8cb31cdd7
+size 5203223845

requirements.txt

@@ -0,0 +1 @@
+git+git://github.com/sapienzanlp/extend