add files

.gitattributes

@@ -25,7 +25,6 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
@@ -33,3 +32,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text
+*.db filter=lfs diff=lfs merge=lfs -text
+

README.md

@@ -1,12 +1,13 @@
 ---
-title: Fact Checking Api
-emoji: 👀
-colorFrom: pink
-colorTo: blue
+title: AVeriTeC
+emoji: 🏆
+colorFrom: purple
+colorTo: red
 sdk: gradio
-sdk_version: 4.38.1
+sdk_version: 4.37.2
 app_file: app.py
 pinned: false
+license: apache-2.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,390 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Created by zd302 at 17/07/2024
+
+from fastapi import FastAPI
+from pydantic import BaseModel
+# from averitec.models.AveritecModule import Wikipediaretriever, Googleretriever, veracity_prediction, justification_generation
+import uvicorn
+
+app = FastAPI()
+
+# ---------------------------------------------------------------------------------------------------------------------
+import os
+import torch
+import numpy as np
+import requests
+from rank_bm25 import BM25Okapi
+from bs4 import BeautifulSoup
+
+from transformers import BartTokenizer, BartForConditionalGeneration
+from transformers import BloomTokenizerFast, BloomForCausalLM, BertTokenizer, BertForSequenceClassification
+from transformers import RobertaTokenizer, RobertaForSequenceClassification
+import pytorch_lightning as pl
+
+from averitec.models.DualEncoderModule import DualEncoderModule
+from averitec.models.SequenceClassificationModule import SequenceClassificationModule
+from averitec.models.JustificationGenerationModule import JustificationGenerationModule
+
+# ---------------------------------------------------------------------------------------------------------------------
+import wikipediaapi
+wiki_wiki = wikipediaapi.Wikipedia('AVeriTeC (zd302@cam.ac.uk)', 'en')
+
+import nltk
+nltk.download('punkt')
+from nltk import pos_tag, word_tokenize, sent_tokenize
+
+import spacy
+os.system("python -m spacy download en_core_web_sm")
+nlp = spacy.load("en_core_web_sm")
+
+# ---------------------------------------------------------------------------------------------------------------------
+# ---------- Load Veracity and Justification prediction model ----------
+LABEL = [
+    "Supported",
+    "Refuted",
+    "Not Enough Evidence",
+    "Conflicting Evidence/Cherrypicking",
+]
+
+# Veracity
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+veracity_tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+bert_model = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=4, problem_type="single_label_classification")
+veracity_checkpoint_path = os.getcwd() + "/averitec/pretrained_models/bert_veracity.ckpt"
+veracity_model = SequenceClassificationModule.load_from_checkpoint(veracity_checkpoint_path,tokenizer=veracity_tokenizer, model=bert_model).to(device)
+# Justification
+justification_tokenizer = BartTokenizer.from_pretrained('facebook/bart-large', add_prefix_space=True)
+bart_model = BartForConditionalGeneration.from_pretrained("facebook/bart-large")
+best_checkpoint = os.getcwd()+ '/averitec/pretrained_models/bart_justifications_verdict-epoch=13-val_loss=2.03-val_meteor=0.28.ckpt'
+justification_model = JustificationGenerationModule.load_from_checkpoint(best_checkpoint, tokenizer=justification_tokenizer, model=bart_model).to(device)
+# ---------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+class Docs:
+    def __init__(self, metadata=dict(), page_content=""):
+        self.metadata = metadata
+        self.page_content = page_content
+
+
+# ------------------------------ Googleretriever -----------------------------
+def Googleretriever():
+
+
+    return 0
+
+# ------------------------------ Googleretriever -----------------------------
+
+# ------------------------------ Wikipediaretriever --------------------------
+def search_entity_wikipeida(entity):
+    find_evidence = []
+
+    page_py = wiki_wiki.page(entity)
+    if page_py.exists():
+        introduction = page_py.summary
+        find_evidence.append([str(entity), introduction])
+
+    return find_evidence
+
+
+def clean_str(p):
+    return p.encode().decode("unicode-escape").encode("latin1").decode("utf-8")
+
+
+def find_similar_wikipedia(entity, relevant_wikipages):
+    # If the relevant wikipeida page of the entity is less than 5, find similar wikipedia pages.
+    ent_ = entity.replace(" ", "+")
+    search_url = f"https://en.wikipedia.org/w/index.php?search={ent_}&title=Special:Search&profile=advanced&fulltext=1&ns0=1"
+    response_text = requests.get(search_url).text
+    soup = BeautifulSoup(response_text, features="html.parser")
+    result_divs = soup.find_all("div", {"class": "mw-search-result-heading"})
+
+    if result_divs:
+        result_titles = [clean_str(div.get_text().strip()) for div in result_divs]
+        similar_titles = result_titles[:5]
+
+        saved_titles = [ent[0] for ent in relevant_wikipages] if relevant_wikipages else relevant_wikipages
+        for _t in similar_titles:
+            if _t not in saved_titles and len(relevant_wikipages) < 5:
+                _evi = search_entity_wikipeida(_t)
+                # _evi = search_step(_t)
+                relevant_wikipages.extend(_evi)
+
+    return relevant_wikipages
+
+
+def find_evidence_from_wikipedia(claim):
+    #
+    doc = nlp(claim)
+    #
+    wikipedia_page = []
+    for ent in doc.ents:
+        relevant_wikipages = search_entity_wikipeida(ent)
+
+        if len(relevant_wikipages) < 5:
+            relevant_wikipages = find_similar_wikipedia(str(ent), relevant_wikipages)
+
+        wikipedia_page.extend(relevant_wikipages)
+
+    return wikipedia_page
+
+
+def bm25_retriever(query, corpus, topk=3):
+    bm25 = BM25Okapi(corpus)
+    #
+    query_tokens = word_tokenize(query)
+    scores = bm25.get_scores(query_tokens)
+    top_n = np.argsort(scores)[::-1][:topk]
+    top_n_scores = [scores[i] for i in top_n]
+
+    return top_n, top_n_scores
+
+
+def relevant_sentence_retrieval(query, wiki_intro, k):
+    # 1. Create corpus here
+    corpus, sentences = [], []
+    titles = []
+    for i, (title, intro) in enumerate(wiki_intro):
+        sents_in_intro = sent_tokenize(intro)
+
+        for sent in sents_in_intro:
+            corpus.append(word_tokenize(sent))
+            sentences.append(sent)
+            titles.append(title)
+
+    # ----- BM25
+    bm25_top_n, bm25_top_n_scores = bm25_retriever(query, corpus, topk=k)
+    bm25_top_n_sents = [sentences[i] for i in bm25_top_n]
+    bm25_top_n_titles = [titles[i] for i in bm25_top_n]
+
+    return bm25_top_n_sents, bm25_top_n_titles
+
+# ------------------------------ Wikipediaretriever -----------------------------
+
+def Wikipediaretriever(claim):
+    # 1. extract relevant wikipedia pages from wikipedia dumps
+    wikipedia_page = find_evidence_from_wikipedia(claim)
+
+    # 2. extract relevant sentences from extracted wikipedia pages
+    sents, titles = relevant_sentence_retrieval(claim, wikipedia_page, k=3)
+
+    #
+    results = []
+    for i, (sent, title) in enumerate(zip(sents, titles)):
+        metadata = dict()
+        metadata['name'] = claim
+        metadata['url'] = "https://en.wikipedia.org/wiki/" + "_".join(title.split())
+        metadata['cached_source_url'] = "https://en.wikipedia.org/wiki/" + "_".join(title)
+        metadata['short_name'] = "Evidence {}".format(i + 1)
+        metadata['page_number'] = ""
+        metadata['query'] = sent
+        metadata['title'] = title
+        metadata['evidence'] = sent
+        metadata['answer'] = ""
+        metadata['page_content'] = "<b>Title</b>: " + str(metadata['title']) + "<br>" + "<b>Evidence</b>: " + metadata['evidence']
+        page_content = f"""{metadata['page_content']}"""
+
+        results.append(Docs(metadata, page_content))
+
+    return results
+
+
+# ------------------------------ Veracity Prediction ------------------------------
+class SequenceClassificationDataLoader(pl.LightningDataModule):
+    def __init__(self, tokenizer, data_file, batch_size, add_extra_nee=False):
+        super().__init__()
+        self.tokenizer = tokenizer
+        self.data_file = data_file
+        self.batch_size = batch_size
+        self.add_extra_nee = add_extra_nee
+
+    def tokenize_strings(
+            self,
+            source_sentences,
+            max_length=400,
+            pad_to_max_length=False,
+            return_tensors="pt",
+    ):
+        encoded_dict = self.tokenizer(
+            source_sentences,
+            max_length=max_length,
+            padding="max_length" if pad_to_max_length else "longest",
+            truncation=True,
+            return_tensors=return_tensors,
+        )
+
+        input_ids = encoded_dict["input_ids"]
+        attention_masks = encoded_dict["attention_mask"]
+
+        return input_ids, attention_masks
+
+    def quadruple_to_string(self, claim, question, answer, bool_explanation=""):
+        if bool_explanation is not None and len(bool_explanation) > 0:
+            bool_explanation = ", because " + bool_explanation.lower().strip()
+        else:
+            bool_explanation = ""
+        return (
+                "[CLAIM] "
+                + claim.strip()
+                + " [QUESTION] "
+                + question.strip()
+                + " "
+                + answer.strip()
+                + bool_explanation
+        )
+
+
+def veracity_prediction(claim, evidence):
+    dataLoader = SequenceClassificationDataLoader(
+        tokenizer=veracity_tokenizer,
+        data_file="this_is_discontinued",
+        batch_size=32,
+        add_extra_nee=False,
+    )
+
+    evidence_strings = []
+    for evi in evidence:
+        evidence_strings.append(dataLoader.quadruple_to_string(claim, evi.metadata["query"], evi.metadata["answer"], ""))
+
+    if len(evidence_strings) == 0:  # If we found no evidence e.g. because google returned 0 pages, just output NEI.
+        pred_label = "Not Enough Evidence"
+        return pred_label
+
+    tokenized_strings, attention_mask = dataLoader.tokenize_strings(evidence_strings)
+    example_support = torch.argmax(
+        veracity_model(tokenized_strings.to(device), attention_mask=attention_mask.to(device)).logits, axis=1)
+
+    has_unanswerable = False
+    has_true = False
+    has_false = False
+
+    for v in example_support:
+        if v == 0:
+            has_true = True
+        if v == 1:
+            has_false = True
+        if v in (2, 3,):  # TODO another hack -- we cant have different labels for train and test so we do this
+            has_unanswerable = True
+
+    if has_unanswerable:
+        answer = 2
+    elif has_true and not has_false:
+        answer = 0
+    elif not has_true and has_false:
+        answer = 1
+    else:
+        answer = 3
+
+    pred_label = LABEL[answer]
+
+    return pred_label
+
+
+# ------------------------------ Justification Generation ------------------------------
+def extract_claim_str(claim, evidence, verdict_label):
+    claim_str = "[CLAIM] " + claim + " [EVIDENCE] "
+
+    for evi in evidence:
+        q_text = evi.metadata['query'].strip()
+
+        if len(q_text) == 0:
+            continue
+
+        if not q_text[-1] == "?":
+            q_text += "?"
+
+        answer_strings = []
+        answer_strings.append(evi.metadata['answer'])
+
+        claim_str += q_text
+        for a_text in answer_strings:
+            if a_text:
+                if not a_text[-1] == ".":
+                    a_text += "."
+                claim_str += " " + a_text.strip()
+
+        claim_str += " "
+
+    claim_str += " [VERDICT] " + verdict_label
+
+    return claim_str
+
+
+def justification_generation(claim, evidence, verdict_label):
+    #
+    claim_str = extract_claim_str(claim, evidence, verdict_label)
+    claim_str.strip()
+    pred_justification = justification_model.generate(claim_str, device=device)
+
+    return pred_justification.strip()
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+class Item(BaseModel):
+    claim: str
+    source: str
+
+
+@app.get("/")
+def greet_json():
+    return {"Hello": "World!"}
+
+
+@app.post("/predict/")
+def fact_checking(item: Item):
+    claim = item['claim']
+    source = item['source']
+    # claim = item.claim
+    # source = item.source
+
+    # Step1: Evidence Retrieval
+    if source == "Wikipedia":
+        evidence = Wikipediaretriever(claim)
+    elif source == "Google":
+        evidence = Googleretriever(claim)
+
+    # Step2: Veracity Prediction and Justification Generation
+    verdict_label = veracity_prediction(claim, evidence)
+    justification_label = justification_generation(claim, evidence, verdict_label)
+
+    evidence_list = []
+    for evi in evidence:
+        evidence_list.append(evi.metadata["query"])
+
+    return  {"Verdict": verdict_label, "Justification": justification_label, "Evidence": evidence_list}
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=7860)
+
+
+# if __name__ == "__main__":
+#     item = {
+#         "claim": "England won the Euro 2024.",
+#         "source": "Wikipedia",
+#     }
+#
+#     results = fact_checking(item)
+#
+#     print(results)
+
+
+
+# # -----------------------------------------------------------------------------------------
+# import requests
+#
+# # 定义API URL
+# api_url = "https://zhenyundeng-zd-api.hf.space/generate/"
+#
+# # 定义请求数据
+# item = {
+#     "name": "Alice"
+# }
+#
+# # 发送Get请求
+# # response = requests.get("https://zhenyundeng-zd-api.hf.space/")
+# # 发送POST请求
+# response = requests.post(api_url, json=item)
+#
+# # 打印响应
+# print(response.json())

averitec/data/all_samples.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef79bab962c2b17d56eb2582b9919bfe8023858fa13ba20c591900857b561854
+size 11444395

averitec/data/sample_claims.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Created by zd302 at 09/05/2024
+
+
+CLAIMS_Type = {
+    "Claim": [
+        "England won the Euro 2024.",
+        "Albert Einstein works in the field of computer science.",
+    ],
+    "Event/Property Claim": [
+        'Hunter Biden had no experience in Ukraine or in the energy sector when he joined the board of Burisma.',
+        "After the police shooting of Jacob Blake, Gov. Tony Evers & Lt. Gov. Mandela Barnes did not call for peace or encourage calm.",
+        "President Trump fully co-operated with the investigation into Russian interference in the 2016 U.S presidential campaign.",
+    ],
+    "Causal Claim":[
+        "Anxiety levels among young teenagers dropped during the coronavirus pandemic, a study has suggested",
+        "Auto workers across Michigan could have lost their jobs if not for Barack Obama and Joe Biden",
+    ],
+    "Numerical Claim":[
+        "Sweden, despite never having had lockdown, has a lower COVID-19 death rate than Spain, Italy, and the United Kingdom.",
+        "According to Harry Roque, even if 10,000 people die, 10 million COVID-19 cases in the country will not be a loss.",
+    ]
+}
+
+CLAIMS_FACT_CHECKING_STRATEGY= {
+    "Written Evidence": [
+        "Pretty Little Thing's terms and conditions state that its products may contain chemicals that can cause cancer, birth defects or other reproductive harm.",
+        "Pretty Little Thing products may contain chemicals that can cause cancer, birth defects or other reproductive harm.",
+    ],
+    "Numerical Comparison":[
+        "Congress party claims regarding shortfall in Government earnings",
+        "On average, one person dies by suicide every 22 hours in West Virginia, United States.",
+    ],
+    "Consultation":[
+        "Your reaction to an optical illusion is an indication of your state of mind.",
+        "The last time people created a Hollywood blacklist, people ended up killing themselves. They were accused, and they lost their right to work.",
+    ]
+}

averitec/models/AveritecModule.py

@@ -0,0 +1,312 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Created by zd302 at 17/07/2024
+
+import torch
+import numpy as np
+import requests
+from rank_bm25 import BM25Okapi
+from bs4 import BeautifulSoup
+
+from transformers import BartTokenizer, BartForConditionalGeneration
+from transformers import BloomTokenizerFast, BloomForCausalLM, BertTokenizer, BertForSequenceClassification
+from transformers import RobertaTokenizer, RobertaForSequenceClassification
+import pytorch_lightning as pl
+
+from averitec.models.DualEncoderModule import DualEncoderModule
+from averitec.models.SequenceClassificationModule import SequenceClassificationModule
+from averitec.models.JustificationGenerationModule import JustificationGenerationModule
+
+
+import wikipediaapi
+wiki_wiki = wikipediaapi.Wikipedia('AVeriTeC (zd302@cam.ac.uk)', 'en')
+import os
+import nltk
+nltk.download('punkt')
+from nltk import pos_tag, word_tokenize, sent_tokenize
+
+import spacy
+os.system("python -m spacy download en_core_web_sm")
+nlp = spacy.load("en_core_web_sm")
+
+# ---------- Load Veracity and Justification prediction model ----------
+LABEL = [
+    "Supported",
+    "Refuted",
+    "Not Enough Evidence",
+    "Conflicting Evidence/Cherrypicking",
+]
+
+# Veracity
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+veracity_tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+bert_model = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=4, problem_type="single_label_classification")
+veracity_model = SequenceClassificationModule.load_from_checkpoint("averitec/pretrained_models/bert_veracity.ckpt",
+                                                                  tokenizer=veracity_tokenizer, model=bert_model).to(device)
+# Justification
+justification_tokenizer = BartTokenizer.from_pretrained('facebook/bart-large', add_prefix_space=True)
+bart_model = BartForConditionalGeneration.from_pretrained("facebook/bart-large")
+best_checkpoint = 'averitec/pretrained_models/bart_justifications_verdict-epoch=13-val_loss=2.03-val_meteor=0.28.ckpt'
+justification_model = JustificationGenerationModule.load_from_checkpoint(best_checkpoint, tokenizer=justification_tokenizer, model=bart_model).to(device)
+# ---------------------------------------------------------------------------
+
+
+# ----------------------------------------------------------------------------
+class Docs:
+    def __init__(self, metadata=dict(), page_content=""):
+        self.metadata = metadata
+        self.page_content = page_content
+
+
+# ------------------------------ Googleretriever -----------------------------
+def Googleretriever():
+
+
+    return 0
+
+# ------------------------------ Googleretriever -----------------------------
+
+# ------------------------------ Wikipediaretriever --------------------------
+def search_entity_wikipeida(entity):
+    find_evidence = []
+
+    page_py = wiki_wiki.page(entity)
+    if page_py.exists():
+        introduction = page_py.summary
+        find_evidence.append([str(entity), introduction])
+
+    return find_evidence
+
+
+def clean_str(p):
+    return p.encode().decode("unicode-escape").encode("latin1").decode("utf-8")
+
+
+def find_similar_wikipedia(entity, relevant_wikipages):
+    # If the relevant wikipeida page of the entity is less than 5, find similar wikipedia pages.
+    ent_ = entity.replace(" ", "+")
+    search_url = f"https://en.wikipedia.org/w/index.php?search={ent_}&title=Special:Search&profile=advanced&fulltext=1&ns0=1"
+    response_text = requests.get(search_url).text
+    soup = BeautifulSoup(response_text, features="html.parser")
+    result_divs = soup.find_all("div", {"class": "mw-search-result-heading"})
+
+    if result_divs:
+        result_titles = [clean_str(div.get_text().strip()) for div in result_divs]
+        similar_titles = result_titles[:5]
+
+        saved_titles = [ent[0] for ent in relevant_wikipages] if relevant_wikipages else relevant_wikipages
+        for _t in similar_titles:
+            if _t not in saved_titles and len(relevant_wikipages) < 5:
+                _evi = search_entity_wikipeida(_t)
+                # _evi = search_step(_t)
+                relevant_wikipages.extend(_evi)
+
+    return relevant_wikipages
+
+
+def find_evidence_from_wikipedia(claim):
+    #
+    doc = nlp(claim)
+    #
+    wikipedia_page = []
+    for ent in doc.ents:
+        relevant_wikipages = search_entity_wikipeida(ent)
+
+        if len(relevant_wikipages) < 5:
+            relevant_wikipages = find_similar_wikipedia(str(ent), relevant_wikipages)
+
+        wikipedia_page.extend(relevant_wikipages)
+
+    return wikipedia_page
+
+
+def bm25_retriever(query, corpus, topk=3):
+    bm25 = BM25Okapi(corpus)
+    #
+    query_tokens = word_tokenize(query)
+    scores = bm25.get_scores(query_tokens)
+    top_n = np.argsort(scores)[::-1][:topk]
+    top_n_scores = [scores[i] for i in top_n]
+
+    return top_n, top_n_scores
+
+
+def relevant_sentence_retrieval(query, wiki_intro, k):
+    # 1. Create corpus here
+    corpus, sentences = [], []
+    titles = []
+    for i, (title, intro) in enumerate(wiki_intro):
+        sents_in_intro = sent_tokenize(intro)
+
+        for sent in sents_in_intro:
+            corpus.append(word_tokenize(sent))
+            sentences.append(sent)
+            titles.append(title)
+
+    # ----- BM25
+    bm25_top_n, bm25_top_n_scores = bm25_retriever(query, corpus, topk=k)
+    bm25_top_n_sents = [sentences[i] for i in bm25_top_n]
+    bm25_top_n_titles = [titles[i] for i in bm25_top_n]
+
+    return bm25_top_n_sents, bm25_top_n_titles
+
+# ------------------------------ Wikipediaretriever -----------------------------
+
+def Wikipediaretriever(claim):
+    # 1. extract relevant wikipedia pages from wikipedia dumps
+    wikipedia_page = find_evidence_from_wikipedia(claim)
+
+    # 2. extract relevant sentences from extracted wikipedia pages
+    sents, titles = relevant_sentence_retrieval(claim, wikipedia_page, k=3)
+
+    #
+    results = []
+    for i, (sent, title) in enumerate(zip(sents, titles)):
+        metadata = dict()
+        metadata['name'] = claim
+        metadata['url'] = "https://en.wikipedia.org/wiki/" + "_".join(title.split())
+        metadata['cached_source_url'] = "https://en.wikipedia.org/wiki/" + "_".join(title)
+        metadata['short_name'] = "Evidence {}".format(i + 1)
+        metadata['page_number'] = ""
+        metadata['query'] = sent
+        metadata['title'] = title
+        metadata['evidence'] = sent
+        metadata['answer'] = ""
+        metadata['page_content'] = "<b>Title</b>: " + str(metadata['title']) + "<br>" + "<b>Evidence</b>: " + metadata['evidence']
+        page_content = f"""{metadata['page_content']}"""
+
+        results.append(Docs(metadata, page_content))
+
+    return results
+
+
+# ------------------------------ Veracity Prediction ------------------------------
+class SequenceClassificationDataLoader(pl.LightningDataModule):
+    def __init__(self, tokenizer, data_file, batch_size, add_extra_nee=False):
+        super().__init__()
+        self.tokenizer = tokenizer
+        self.data_file = data_file
+        self.batch_size = batch_size
+        self.add_extra_nee = add_extra_nee
+
+    def tokenize_strings(
+            self,
+            source_sentences,
+            max_length=400,
+            pad_to_max_length=False,
+            return_tensors="pt",
+    ):
+        encoded_dict = self.tokenizer(
+            source_sentences,
+            max_length=max_length,
+            padding="max_length" if pad_to_max_length else "longest",
+            truncation=True,
+            return_tensors=return_tensors,
+        )
+
+        input_ids = encoded_dict["input_ids"]
+        attention_masks = encoded_dict["attention_mask"]
+
+        return input_ids, attention_masks
+
+    def quadruple_to_string(self, claim, question, answer, bool_explanation=""):
+        if bool_explanation is not None and len(bool_explanation) > 0:
+            bool_explanation = ", because " + bool_explanation.lower().strip()
+        else:
+            bool_explanation = ""
+        return (
+                "[CLAIM] "
+                + claim.strip()
+                + " [QUESTION] "
+                + question.strip()
+                + " "
+                + answer.strip()
+                + bool_explanation
+        )
+
+
+def veracity_prediction(claim, evidence):
+    dataLoader = SequenceClassificationDataLoader(
+        tokenizer=veracity_tokenizer,
+        data_file="this_is_discontinued",
+        batch_size=32,
+        add_extra_nee=False,
+    )
+
+    evidence_strings = []
+    for evi in evidence:
+        evidence_strings.append(dataLoader.quadruple_to_string(claim, evi.metadata["query"], evi.metadata["answer"], ""))
+
+    if len(evidence_strings) == 0:  # If we found no evidence e.g. because google returned 0 pages, just output NEI.
+        pred_label = "Not Enough Evidence"
+        return pred_label
+
+    tokenized_strings, attention_mask = dataLoader.tokenize_strings(evidence_strings)
+    example_support = torch.argmax(
+        veracity_model(tokenized_strings.to(device), attention_mask=attention_mask.to(device)).logits, axis=1)
+
+    has_unanswerable = False
+    has_true = False
+    has_false = False
+
+    for v in example_support:
+        if v == 0:
+            has_true = True
+        if v == 1:
+            has_false = True
+        if v in (2, 3,):  # TODO another hack -- we cant have different labels for train and test so we do this
+            has_unanswerable = True
+
+    if has_unanswerable:
+        answer = 2
+    elif has_true and not has_false:
+        answer = 0
+    elif not has_true and has_false:
+        answer = 1
+    else:
+        answer = 3
+
+    pred_label = LABEL[answer]
+
+    return pred_label
+
+
+# ------------------------------ Justification Generation ------------------------------
+def extract_claim_str(claim, evidence, verdict_label):
+    claim_str = "[CLAIM] " + claim + " [EVIDENCE] "
+
+    for evi in evidence:
+        q_text = evi.metadata['query'].strip()
+
+        if len(q_text) == 0:
+            continue
+
+        if not q_text[-1] == "?":
+            q_text += "?"
+
+        answer_strings = []
+        answer_strings.append(evi.metadata['answer'])
+
+        claim_str += q_text
+        for a_text in answer_strings:
+            if a_text:
+                if not a_text[-1] == ".":
+                    a_text += "."
+                claim_str += " " + a_text.strip()
+
+        claim_str += " "
+
+    claim_str += " [VERDICT] " + verdict_label
+
+    return claim_str
+
+
+def justification_generation(claim, evidence, verdict_label):
+    #
+    claim_str = extract_claim_str(claim, evidence, verdict_label)
+    claim_str.strip()
+    pred_justification = justification_model.generate(claim_str, device=device)
+
+    return pred_justification.strip()
+
+

averitec/models/DualEncoderModule.py

@@ -0,0 +1,143 @@
+import pytorch_lightning as pl
+import torch
+from transformers.optimization import AdamW
+import torchmetrics
+
+
+class DualEncoderModule(pl.LightningModule):
+
+    def __init__(self, tokenizer, model, learning_rate=1e-3):
+        super().__init__()
+        self.tokenizer = tokenizer
+        self.model = model
+        self.learning_rate = learning_rate
+
+        self.train_acc = torchmetrics.Accuracy(
+            task="multiclass", num_classes=model.num_labels
+        )
+        self.val_acc = torchmetrics.Accuracy(
+            task="multiclass", num_classes=model.num_labels
+        )
+        self.test_acc = torchmetrics.Accuracy(
+            task="multiclass", num_classes=model.num_labels
+        )
+
+    def forward(self, input_ids, **kwargs):
+        return self.model(input_ids, **kwargs)
+
+    def configure_optimizers(self):
+        optimizer = AdamW(self.parameters(), lr=self.learning_rate)
+        return optimizer
+
+    def training_step(self, batch, batch_idx):
+        pos_ids, pos_mask, neg_ids, neg_mask = batch
+
+        neg_ids = neg_ids.view(-1, neg_ids.shape[-1])
+        neg_mask = neg_mask.view(-1, neg_mask.shape[-1])
+
+        pos_outputs = self(
+            pos_ids,
+            attention_mask=pos_mask,
+            labels=torch.ones(pos_ids.shape[0], dtype=torch.uint8).to(
+                pos_ids.get_device()
+            ),
+        )
+        neg_outputs = self(
+            neg_ids,
+            attention_mask=neg_mask,
+            labels=torch.zeros(neg_ids.shape[0], dtype=torch.uint8).to(
+                neg_ids.get_device()
+            ),
+        )
+
+        loss_scale = 1.0
+        loss = pos_outputs.loss + loss_scale * neg_outputs.loss
+
+        pos_logits = pos_outputs.logits
+        pos_preds = torch.argmax(pos_logits, axis=1)
+        self.train_acc(
+            pos_preds.cpu(), torch.ones(pos_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        neg_logits = neg_outputs.logits
+        neg_preds = torch.argmax(neg_logits, axis=1)
+        self.train_acc(
+            neg_preds.cpu(), torch.zeros(neg_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        return {"loss": loss}
+
+    def validation_step(self, batch, batch_idx):
+        pos_ids, pos_mask, neg_ids, neg_mask = batch
+
+        neg_ids = neg_ids.view(-1, neg_ids.shape[-1])
+        neg_mask = neg_mask.view(-1, neg_mask.shape[-1])
+
+        pos_outputs = self(
+            pos_ids,
+            attention_mask=pos_mask,
+            labels=torch.ones(pos_ids.shape[0], dtype=torch.uint8).to(
+                pos_ids.get_device()
+            ),
+        )
+        neg_outputs = self(
+            neg_ids,
+            attention_mask=neg_mask,
+            labels=torch.zeros(neg_ids.shape[0], dtype=torch.uint8).to(
+                neg_ids.get_device()
+            ),
+        )
+
+        loss_scale = 1.0
+        loss = pos_outputs.loss + loss_scale * neg_outputs.loss
+
+        pos_logits = pos_outputs.logits
+        pos_preds = torch.argmax(pos_logits, axis=1)
+        self.val_acc(
+            pos_preds.cpu(), torch.ones(pos_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        neg_logits = neg_outputs.logits
+        neg_preds = torch.argmax(neg_logits, axis=1)
+        self.val_acc(
+            neg_preds.cpu(), torch.zeros(neg_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        self.log("val_acc", self.val_acc)
+
+        return {"loss": loss}
+
+    def test_step(self, batch, batch_idx):
+        pos_ids, pos_mask, neg_ids, neg_mask = batch
+
+        neg_ids = neg_ids.view(-1, neg_ids.shape[-1])
+        neg_mask = neg_mask.view(-1, neg_mask.shape[-1])
+
+        pos_outputs = self(
+            pos_ids,
+            attention_mask=pos_mask,
+            labels=torch.ones(pos_ids.shape[0], dtype=torch.uint8).to(
+                pos_ids.get_device()
+            ),
+        )
+        neg_outputs = self(
+            neg_ids,
+            attention_mask=neg_mask,
+            labels=torch.zeros(neg_ids.shape[0], dtype=torch.uint8).to(
+                neg_ids.get_device()
+            ),
+        )
+
+        pos_logits = pos_outputs.logits
+        pos_preds = torch.argmax(pos_logits, axis=1)
+        self.test_acc(
+            pos_preds.cpu(), torch.ones(pos_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        neg_logits = neg_outputs.logits
+        neg_preds = torch.argmax(neg_logits, axis=1)
+        self.test_acc(
+            neg_preds.cpu(), torch.zeros(neg_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        self.log("test_acc", self.test_acc)

averitec/models/JustificationGenerationModule.py

@@ -0,0 +1,193 @@
+import pytorch_lightning as pl
+import torch
+import numpy as np
+import datasets
+from transformers import MaxLengthCriteria, StoppingCriteriaList
+from transformers.optimization import AdamW
+import itertools
+from averitec.models.utils import count_stats, f1_metric, pairwise_meteor
+from torchmetrics.text.rouge import ROUGEScore
+import torch.nn.functional as F
+import torchmetrics
+from torchmetrics.classification import F1Score
+
+def freeze_params(model):
+  for layer in model.parameters():
+    layer.requires_grade = False
+
+class JustificationGenerationModule(pl.LightningModule):
+    
+  def __init__(self, tokenizer, model, learning_rate=1e-3, gen_num_beams=2, gen_max_length=100, should_pad_gen=True):
+    super().__init__()
+    self.tokenizer = tokenizer
+    self.model = model
+    self.learning_rate = learning_rate
+
+    self.gen_num_beams = gen_num_beams
+    self.gen_max_length = gen_max_length
+    self.should_pad_gen = should_pad_gen
+
+    #self.metrics =  datasets.load_metric('meteor')
+
+    freeze_params(self.model.get_encoder())
+    self.freeze_embeds()
+  
+  def freeze_embeds(self):
+    ''' freeze the positional embedding parameters of the model; adapted from finetune.py '''
+    freeze_params(self.model.model.shared)
+    for d in [self.model.model.encoder, self.model.model.decoder]:
+      freeze_params(d.embed_positions)
+      freeze_params(d.embed_tokens)
+
+  # Do a forward pass through the model
+  def forward(self, input_ids, **kwargs):
+    return self.model(input_ids, **kwargs)
+  
+  def configure_optimizers(self):
+    optimizer = AdamW(self.parameters(), lr = self.learning_rate)
+    return optimizer
+
+  def shift_tokens_right(self, input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):
+    """
+    Shift input ids one token to the right.
+    https://github.com/huggingface/transformers/blob/main/src/transformers/models/bart/modeling_bart.py.
+    """
+    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
+    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
+    shifted_input_ids[:, 0] = decoder_start_token_id
+
+    if pad_token_id is None:
+        raise ValueError("self.model.config.pad_token_id has to be defined.")
+    # replace possible -100 values in labels by `pad_token_id`
+    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)
+
+    return shifted_input_ids
+
+  def run_model(self, batch):
+    src_ids, src_mask, tgt_ids = batch[0], batch[1], batch[2]
+
+    decoder_input_ids = self.shift_tokens_right(
+                tgt_ids, self.tokenizer.pad_token_id, self.tokenizer.pad_token_id # BART uses the EOS token to start generation as well. Might have to change for other models.
+            )
+
+    outputs = self(src_ids, attention_mask=src_mask, decoder_input_ids=decoder_input_ids, use_cache=False)
+    return outputs
+
+  def compute_loss(self, batch):
+    tgt_ids = batch[2]
+    logits = self.run_model(batch)[0]
+
+    cross_entropy = torch.nn.CrossEntropyLoss(ignore_index=self.tokenizer.pad_token_id)
+    loss = cross_entropy(logits.view(-1, logits.shape[-1]), tgt_ids.view(-1))
+
+    return loss
+
+  def training_step(self, batch, batch_idx):
+    loss = self.compute_loss(batch)
+
+    self.log("train_loss", loss, on_epoch=True)
+
+    return {'loss':loss}
+
+  def validation_step(self, batch, batch_idx):
+    preds, loss, tgts = self.generate_and_compute_loss_and_tgts(batch)
+    if self.should_pad_gen:
+      preds = F.pad(preds, pad=(0, self.gen_max_length - preds.shape[1]), value=self.tokenizer.pad_token_id)
+
+    self.log('val_loss', loss, prog_bar=True, sync_dist=True)
+
+    return {'loss': loss, 'pred': preds, 'target': tgts}
+
+  def test_step(self, batch, batch_idx):
+    test_preds, test_loss, test_tgts = self.generate_and_compute_loss_and_tgts(batch)
+    if self.should_pad_gen:
+      test_preds = F.pad(test_preds, pad=(0, self.gen_max_length - test_preds.shape[1]), value=self.tokenizer.pad_token_id)
+
+    self.log('test_loss', test_loss, prog_bar=True, sync_dist=True)
+
+    return {'loss': test_loss, 'pred': test_preds, 'target': test_tgts}
+
+  def test_epoch_end(self, outputs):
+    self.handle_end_of_epoch_scoring(outputs, "test")
+
+  def validation_epoch_end(self, outputs):
+    self.handle_end_of_epoch_scoring(outputs, "val")
+
+  def handle_end_of_epoch_scoring(self, outputs, prefix):
+      gen = {}
+      tgt = {}
+      rouge = ROUGEScore()
+      rouge_metric = lambda x, y: rouge(x,y)["rougeL_precision"]
+      for out in outputs:
+        preds = out['pred']
+        tgts = out['target']
+
+        preds = self.do_batch_detokenize(preds)
+        tgts = self.do_batch_detokenize(tgts)
+
+        for pred, t in zip(preds, tgts):
+          rouge_d = rouge_metric(pred, t)
+          self.log(prefix+"_rouge", rouge_d)
+
+          meteor_d = pairwise_meteor(pred, t)
+          self.log(prefix+"_meteor", meteor_d)
+
+  def generate_and_compute_loss_and_tgts(self, batch):
+    src_ids = batch[0]
+    loss = self.compute_loss(batch)
+    pred_ids, _ = self.generate_for_batch(src_ids)
+
+    tgt_ids = batch[2]
+
+    return pred_ids, loss, tgt_ids
+
+  def do_batch_detokenize(self, batch):
+    tokens = self.tokenizer.batch_decode(
+      batch, 
+      skip_special_tokens=True, 
+      clean_up_tokenization_spaces=True
+      )
+
+    # Huggingface skipping of special tokens doesn't work for all models, so we do it manually as well for safety:
+    tokens = [p.replace("<pad>", "") for p in tokens]
+    tokens = [p.replace("<s>", "") for p in tokens]
+    tokens = [p.replace("</s>", "") for p in tokens]
+
+    return [t for t in tokens if t != ""]
+  
+  def generate_for_batch(self, batch):
+    generated_ids = self.model.generate(
+      batch, 
+      decoder_start_token_id = self.tokenizer.pad_token_id,
+      num_beams = self.gen_num_beams,
+      max_length = self.gen_max_length
+      )
+
+    generated_tokens = self.tokenizer.batch_decode(
+      generated_ids, 
+      skip_special_tokens=True, 
+      clean_up_tokenization_spaces=True
+      )
+
+    return generated_ids, generated_tokens
+
+
+  def generate(self, text, max_input_length=512, device=None):
+    encoded_dict = self.tokenizer(
+            [text],
+            max_length=max_input_length,
+            padding="longest",
+            truncation=True,
+            return_tensors="pt",
+            add_prefix_space = True
+        )
+
+    input_ids = encoded_dict['input_ids']
+
+    if device is not None:
+      input_ids = input_ids.to(device)
+
+    with torch.no_grad():
+        _, generated_tokens = self.generate_for_batch(input_ids)
+    
+    return generated_tokens[0]

averitec/models/NaiveSeqClassModule.py

@@ -0,0 +1,145 @@
+import pytorch_lightning as pl
+import torch
+import numpy as np
+import datasets
+from transformers import MaxLengthCriteria, StoppingCriteriaList
+from transformers.optimization import AdamW
+import itertools
+from utils import count_stats, f1_metric, pairwise_meteor
+from torchmetrics.text.rouge import ROUGEScore
+import torch.nn.functional as F
+import torchmetrics
+from torchmetrics.classification import F1Score
+
+class NaiveSeqClassModule(pl.LightningModule):
+  # Instantiate the model
+  def __init__(self, tokenizer, model, use_question_stance_approach=True, learning_rate=1e-3):
+    super().__init__()
+    self.tokenizer = tokenizer
+    self.model = model
+    self.learning_rate = learning_rate
+
+    self.train_acc = torchmetrics.Accuracy()
+    self.val_acc = torchmetrics.Accuracy()
+    self.test_acc = torchmetrics.Accuracy()
+
+    self.train_f1 = F1Score(num_classes=4, average="macro")
+    self.val_f1 = F1Score(num_classes=4, average=None)
+    self.test_f1 = F1Score(num_classes=4, average=None)
+
+    self.use_question_stance_approach = use_question_stance_approach
+
+  
+  # Do a forward pass through the model
+  def forward(self, input_ids, **kwargs):
+    return self.model(input_ids, **kwargs)
+  
+  def configure_optimizers(self):
+    optimizer = AdamW(self.parameters(), lr = self.learning_rate)
+    return optimizer
+
+  def training_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+
+    outputs = self(x, attention_mask=x_mask, labels=y)
+    logits = outputs.logits
+    loss = outputs.loss
+
+    #cross_entropy = torch.nn.CrossEntropyLoss()
+    #loss = cross_entropy(logits, y)
+    
+    preds = torch.argmax(logits, axis=1)
+
+    self.train_acc(preds.cpu(), y.cpu())      
+    self.train_f1(preds.cpu(), y.cpu())      
+    
+    self.log("train_loss", loss)
+
+    return {'loss': loss}
+
+  def training_epoch_end(self, outs):
+    self.log('train_acc_epoch', self.train_acc)
+    self.log('train_f1_epoch', self.train_f1)
+
+  def validation_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+
+    outputs = self(x, attention_mask=x_mask, labels=y)
+    logits = outputs.logits
+    loss = outputs.loss
+
+    preds = torch.argmax(logits, axis=1)
+
+    if not self.use_question_stance_approach:
+      self.val_acc(preds, y)
+      self.log('val_acc_step', self.val_acc)
+
+      self.val_f1(preds, y)      
+      self.log("val_loss", loss)
+
+    return {'val_loss':loss, "src": x, "pred": preds, "target": y}
+
+  def validation_epoch_end(self, outs):
+    if self.use_question_stance_approach:
+      self.handle_end_of_epoch_scoring(outs, self.val_acc, self.val_f1)
+    
+    self.log('val_acc_epoch', self.val_acc)
+
+    f1 = self.val_f1.compute()
+    self.val_f1.reset()
+
+    self.log('val_f1_epoch', torch.mean(f1))
+
+    class_names = ["supported", "refuted", "nei", "conflicting"]
+    for i, c_name in enumerate(class_names):
+      self.log("val_f1_" + c_name, f1[i])
+
+
+  def test_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+    
+    outputs = self(x, attention_mask=x_mask)
+    logits = outputs.logits
+
+    preds = torch.argmax(logits, axis=1)
+    
+    if not self.use_question_stance_approach:
+      self.test_acc(preds, y)      
+      self.log('test_acc_step', self.test_acc)
+      self.test_f1(preds, y)      
+
+    return {"src": x, "pred": preds, "target": y}
+
+  def test_epoch_end(self, outs):
+    if self.use_question_stance_approach:
+      self.handle_end_of_epoch_scoring(outs, self.test_acc, self.test_f1)
+    
+    self.log('test_acc_epoch', self.test_acc)
+
+    f1 = self.test_f1.compute()
+    self.test_f1.reset()
+    self.log('test_f1_epoch', torch.mean(f1))
+
+    class_names = ["supported", "refuted", "nei", "conflicting"]
+    for i, c_name in enumerate(class_names):
+      self.log("test_f1_" + c_name, f1[i])
+
+  def handle_end_of_epoch_scoring(self, outputs, acc_scorer, f1_scorer):
+      gold_labels = {}
+      question_support = {}
+      for out in outputs:
+        srcs = out['src']
+        preds = out['pred']
+        tgts = out['target']
+
+        tokens = self.tokenizer.batch_decode(
+          srcs, 
+          skip_special_tokens=True, 
+          clean_up_tokenization_spaces=True
+        )
+
+        for src, pred, tgt in zip(tokens, preds, tgts):
+          acc_scorer(torch.as_tensor([pred]).to("cuda:0"), torch.as_tensor([tgt]).to("cuda:0"))    
+          f1_scorer(torch.as_tensor([pred]).to("cuda:0"), torch.as_tensor([tgt]).to("cuda:0"))
+
+        

averitec/models/SequenceClassificationModule.py

@@ -0,0 +1,179 @@
+import pytorch_lightning as pl
+import torch
+import numpy as np
+import datasets
+from transformers import MaxLengthCriteria, StoppingCriteriaList
+from transformers.optimization import AdamW
+import itertools
+# from utils import count_stats, f1_metric, pairwise_meteor
+from torchmetrics.text.rouge import ROUGEScore
+import torch.nn.functional as F
+import torchmetrics
+from torchmetrics.classification import F1Score
+
+class SequenceClassificationModule(pl.LightningModule):
+  # Instantiate the model
+  def __init__(self, tokenizer, model, use_question_stance_approach=True, learning_rate=1e-3):
+    super().__init__()
+    self.tokenizer = tokenizer
+    self.model = model
+    self.learning_rate = learning_rate
+
+    self.train_acc = torchmetrics.Accuracy(task="multiclass", num_classes=model.num_labels)
+    self.val_acc = torchmetrics.Accuracy(task="multiclass", num_classes=model.num_labels)
+    self.test_acc = torchmetrics.Accuracy(task="multiclass", num_classes=model.num_labels)
+
+    self.train_f1 = F1Score(task="multiclass", num_classes=model.num_labels, average="macro")
+    self.val_f1 = F1Score(task="multiclass", num_classes=model.num_labels, average=None)
+    self.test_f1 = F1Score(task="multiclass", num_classes=model.num_labels, average=None)
+    # self.train_acc = torchmetrics.Accuracy()
+    # self.val_acc = torchmetrics.Accuracy()
+    # self.test_acc = torchmetrics.Accuracy()
+
+    # self.train_f1 = F1Score(num_classes=4, average="macro")
+    # self.val_f1 = F1Score(num_classes=4, average=None)
+    # self.test_f1 = F1Score(num_classes=4, average=None)
+
+    self.use_question_stance_approach = use_question_stance_approach
+
+  
+  # Do a forward pass through the model
+  def forward(self, input_ids, **kwargs):
+    return self.model(input_ids, **kwargs)
+  
+  def configure_optimizers(self):
+    optimizer = AdamW(self.parameters(), lr = self.learning_rate)
+    return optimizer
+
+  def training_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+
+    outputs = self(x, attention_mask=x_mask, labels=y)
+    logits = outputs.logits
+    loss = outputs.loss
+
+    #cross_entropy = torch.nn.CrossEntropyLoss()
+    #loss = cross_entropy(logits, y)
+    
+    preds = torch.argmax(logits, axis=1)
+
+    self.log("train_loss", loss)
+
+    return {'loss': loss}
+
+  def validation_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+
+    outputs = self(x, attention_mask=x_mask, labels=y)
+    logits = outputs.logits
+    loss = outputs.loss
+
+    preds = torch.argmax(logits, axis=1)
+
+    if not self.use_question_stance_approach:
+      self.val_acc(preds, y)
+      self.log('val_acc_step', self.val_acc)
+
+      self.val_f1(preds, y)      
+      self.log("val_loss", loss)
+
+    return {'val_loss':loss, "src": x, "pred": preds, "target": y}
+
+  def validation_epoch_end(self, outs):
+    if self.use_question_stance_approach:
+      self.handle_end_of_epoch_scoring(outs, self.val_acc, self.val_f1)
+    
+    self.log('val_acc_epoch', self.val_acc)
+
+    f1 = self.val_f1.compute()
+    self.val_f1.reset()
+
+    self.log('val_f1_epoch', torch.mean(f1))
+
+    class_names = ["supported", "refuted", "nei", "conflicting"]
+    for i, c_name in enumerate(class_names):
+      self.log("val_f1_" + c_name, f1[i])
+
+
+  def test_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+    
+    outputs = self(x, attention_mask=x_mask)
+    logits = outputs.logits
+
+    preds = torch.argmax(logits, axis=1)
+    
+    if not self.use_question_stance_approach:
+      self.test_acc(preds, y)      
+      self.log('test_acc_step', self.test_acc)
+      self.test_f1(preds, y)      
+
+    return {"src": x, "pred": preds, "target": y}
+
+  def test_epoch_end(self, outs):
+    if self.use_question_stance_approach:
+      self.handle_end_of_epoch_scoring(outs, self.test_acc, self.test_f1)
+    
+    self.log('test_acc_epoch', self.test_acc)
+
+    f1 = self.test_f1.compute()
+    self.test_f1.reset()
+    self.log('test_f1_epoch', torch.mean(f1))
+
+    class_names = ["supported", "refuted", "nei", "conflicting"]
+    for i, c_name in enumerate(class_names):
+      self.log("test_f1_" + c_name, f1[i])
+
+  def handle_end_of_epoch_scoring(self, outputs, acc_scorer, f1_scorer):
+      gold_labels = {}
+      question_support = {}
+      for out in outputs:
+        srcs = out['src']
+        preds = out['pred']
+        tgts = out['target']
+
+        tokens = self.tokenizer.batch_decode(
+          srcs, 
+          skip_special_tokens=True, 
+          clean_up_tokenization_spaces=True
+        )
+
+        for src, pred, tgt in zip(tokens, preds, tgts):
+          claim_id = src.split("[ question ]")[0]
+
+          if claim_id not in gold_labels:
+            gold_labels[claim_id] = tgt
+            question_support[claim_id] = []
+
+          question_support[claim_id].append(pred)
+
+      for k,gold_label in gold_labels.items():
+        support = question_support[k]
+
+        has_unansw = False
+        has_true = False
+        has_false = False
+
+        for v in support:
+          if v == 0:
+            has_true = True
+          if v == 1:
+            has_false = True
+          if v == 2 or v == 3: # TODO very ugly hack -- we cant have different numbers of labels for train and test so we do this
+            has_unansw = True
+
+        if has_unansw:
+          answer = 2
+        elif has_true and not has_false:
+          answer = 0
+        elif has_false and not has_true:
+          answer = 1
+        elif has_true and has_false:
+          answer = 3
+
+
+        # TODO this is very hacky and wont work if the device is literally anything other than cuda:0
+        acc_scorer(torch.as_tensor([answer]).to("cuda:0"), torch.as_tensor([gold_label]).to("cuda:0"))    
+        f1_scorer(torch.as_tensor([answer]).to("cuda:0"), torch.as_tensor([gold_label]).to("cuda:0"))
+
+        

averitec/models/utils.py

@@ -0,0 +1,119 @@
+import os
+import nltk
+from nltk import word_tokenize
+import numpy as np
+from leven import levenshtein
+from sklearn.cluster import DBSCAN, dbscan
+
+def delete_if_exists(filepath):
+    if os.path.exists(filepath):
+        os.remove(filepath)
+
+def pairwise_meteor(candidate, reference): # Todo this is not thread safe, no idea how to make it so
+    return nltk.translate.meteor_score.single_meteor_score(word_tokenize(reference), word_tokenize(candidate))
+
+def count_stats(candidate_dict, reference_dict):
+    count_match = [0 for _ in candidate_dict]
+    count_diff = [0 for _ in candidate_dict]
+
+    for i, k in enumerate(candidate_dict.keys()):
+      pred_parts = candidate_dict[k]
+      tgt_parts = reference_dict[k]
+
+      if len(pred_parts) == len(tgt_parts):
+        count_match[i] = 1
+
+      count_diff[i] = abs(len(pred_parts) - len(tgt_parts))
+
+    count_match_score = np.mean(count_match)
+    count_diff_score = np.mean(count_diff)
+
+    return {
+        "count_match_score": count_match_score,
+        "count_diff_score": count_diff_score
+    }
+
+def f1_metric(candidate_dict, reference_dict, pairwise_metric):
+    all_best_p = [0 for _ in candidate_dict]
+    all_best_t = [0 for _ in candidate_dict]
+    p_unnorm = []
+
+    for i, k in enumerate(candidate_dict.keys()):
+      pred_parts = candidate_dict[k]
+      tgt_parts = reference_dict[k]
+
+      best_p_score = [0 for _ in pred_parts]
+      best_t_score = [0 for _ in tgt_parts]
+
+      for p_idx in range(len(pred_parts)):
+        for t_idx in range(len(tgt_parts)):
+          #meteor_score = pairwise_meteor(pred_parts[p_idx], tgt_parts[t_idx])
+          metric_score = pairwise_metric(pred_parts[p_idx], tgt_parts[t_idx])
+
+          if metric_score > best_p_score[p_idx]:
+            best_p_score[p_idx] = metric_score
+
+          if metric_score > best_t_score[t_idx]:
+            best_t_score[t_idx] = metric_score
+
+      all_best_p[i] = np.mean(best_p_score) if len(best_p_score) > 0 else 1.0
+      all_best_t[i] = np.mean(best_t_score) if len(best_t_score) > 0 else 1.0       
+
+      p_unnorm.extend(best_p_score) 
+
+    p_score = np.mean(all_best_p)
+    r_score = np.mean(all_best_t)
+    avg_score = (p_score + r_score) / 2
+    f1_score = 2 * p_score * r_score / (p_score + r_score + 1e-8)
+
+    p_unnorm_score = np.mean(p_unnorm)
+
+    return {
+        "p": p_score,
+        "r": r_score,
+        "avg": avg_score,
+        "f1": f1_score,
+        "p_unnorm": p_unnorm_score,
+    }
+
+def edit_distance_dbscan(data):
+  # Inspired by https://scikit-learn.org/stable/faq.html#how-do-i-deal-with-string-data-or-trees-graphs
+  def lev_metric(x, y):
+    i, j = int(x[0]), int(y[0])
+    return levenshtein(data[i], data[j])
+
+  X = np.arange(len(data)).reshape(-1, 1)
+
+  clustering = dbscan(X, metric=lev_metric, eps=20, min_samples=2, algorithm='brute')
+  return clustering
+
+def compute_all_pairwise_edit_distances(data):
+  X = np.empty((len(data), len(data)))
+
+  for i in range(len(data)):
+    for j in range(len(data)):
+      X[i][j] = levenshtein(data[i], data[j])
+
+  return X
+
+def compute_all_pairwise_scores(src_data, tgt_data, metric):
+  X = np.empty((len(src_data), len(tgt_data)))
+
+  for i in range(len(src_data)):
+    for j in range(len(tgt_data)):
+      X[i][j] = (metric(src_data[i], tgt_data[j]))
+
+  return X
+
+def compute_all_pairwise_meteor_scores(data):
+  X = np.empty((len(data), len(data)))
+
+  for i in range(len(data)):
+    for j in range(len(data)):
+      X[i][j] = (pairwise_meteor(data[i], data[j]) + pairwise_meteor(data[j], data[i])) / 2
+
+  return X
+
+def edit_distance_custom(data, X, eps=0.5, min_samples=3):
+  clustering = DBSCAN(metric="precomputed", eps=eps, min_samples=min_samples).fit(X)
+  return clustering.labels_

averitec/pretrained_models/bart_justifications_verdict-epoch=13-val_loss=2.03-val_meteor=0.28.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4b7bf02daaf10b3443f4f2cbe79c3c9f10c453dfdf818a4d14e44b2b4311cf4
+size 4876206567

averitec/pretrained_models/bert_dual_encoder.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fee6737f655f4f1dfb46cc1bb812b5eaf9a72cfc0b69d4e5c05cde27ea7b6051
+size 1314015751

averitec/pretrained_models/bert_veracity.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ddb8132a28ceff149904dd3ad3c3edd3e5f0c7de0169819207104a80e425c9a
+size 1314034311

requirements.txt

@@ -0,0 +1,22 @@
+gradio
+nltk
+rank_bm25
+accelerate
+trafilatura
+spacy
+pytorch_lightning
+transformers==4.29.2
+datasets
+leven
+scikit-learn
+pexpect
+elasticsearch
+torch
+huggingface_hub
+google-api-python-client
+wikipedia-api
+beautifulsoup4
+azure-storage-file-share
+azure-storage-blob
+bm25s
+PyStemmer