Upload 5 files

datasets/hintqa.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c56b23d510456dca12e37b47e5b4337b9759c66e34214ba6bdc4745f5b3e8ce1
-size 55383770
+oid sha256:f8e58f86e93367caaf2508047fa2e5859a60754ad4811c30247e53d61746eeb3
+size 55383750

datasets/kg-hint.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d40ba25c2ed48661d708f440423d0cb84224266128fc52aa705970871650bcec
+oid sha256:ed988defd1f612016e485853206ead60c33d38dddb319aaf1ed335141335e67b
 size 66070

datasets/metadata.json

@@ -1 +1 @@
-{"hintqa": {"name": "HintQA", "version": "1.0", "description": "This dataset is generated using the method proposed in the TriviaHG paper for the test set questions of TriviaQA, NQ, and WebQ datasets. For each question, up to seven hints are generated and evaluated using various metrics.", "url": "https://github.com/DataScienceUIBK/HintQA", "subsets": {"WebQ-Vanilla": {"questions": 2032, "hints": 15812}, "WebQ-Finetuned": {"questions": 2032, "hints": 16978}, "NQ-Vanilla": {"questions": 3610, "hints": 30976}, "NQ-Finetuned": {"questions": 3610, "hints": 33131}, "TriviaQA-Vanilla": {"questions": 11313, "hints": 103018}, "TriviaQA-Finetuned": {"questions": 11313, "hints": 105709}}}, "kg-hint": {"name": "KG-Hint", "version": "1.0", "description": "This dataset is generated using the AutomaticHintGeneration system. The system processes an .xlsx file containing the question, answer, and category. It retrieves information related to the answer from Wikidata and Wikipedia to generate hints. The answer must be a Wikipedia entity and can be a person, year, or location.", "url": "https://dl.acm.org/doi/10.1145/3578337.3605119", "subsets": {"entire": {"questions": 30, "hints": 307}}}, "triviahg": {"name": "TriviaHG", "version": "1.0", "description": "TriviaHG is an extensive dataset crafted specifically for hint generation in question answering.", "url": "https://github.com/DataScienceUIBK/TriviaHG", "subsets": {"LLaMA_13b_Vanilla": {"questions": 100, "hints": 896}, "WizardLM_70b": {"questions": 100, "hints": 941}, "GPT_4": {"questions": 100, "hints": 949}, "Copilot": {"questions": 100, "hints": 970}, "LLaMA_70b_Finetuned": {"questions": 100, "hints": 924}, "LLaMA_70b_Vanilla": {"questions": 100, "hints": 683}, "Gemini": {"questions": 100, "hints": 942}, "LLaMA_7b_Finetuned": {"questions": 100, "hints": 923}, "LLaMA_7b_Vanilla": {"questions": 100, "hints": 840}, "LLaMA_13b_Finetuned": {"questions": 100, "hints": 921}, "GPT_3.5": {"questions": 100, "hints": 898}, "test": {"questions": 1000, "hints": 9617}, "validation": {"questions": 1000, "hints": 9638}, "training": {"questions": 14645, "hints": 140973}}}, "wikihint": {"name": "WikiHint", "version": "1.0", "description": "The dataset was created using Amazon Mechanical Turk (MTurk). Workers were instructed to write five hints based on a Wikipedia article as their knowledge base. After writing all five hints, they ranked their hints by usefulness. A hint is ranked higher if it is more likely to lead the user to the correct answer. The best hint is assigned a rank of 1, and the worst hint is assigned a rank of 5.", "url": "https://github.com/FloGerhold/Hint_Dataset", "subsets": {"train": {"questions": 900, "hints": 4500}, "test": {"questions": 100, "hints": 500}, "LLaMA-3.1-8b-FTwA-answer-aware": {"questions": 100, "hints": 100}, "GPT-4-Vanilla-answer-agnostic": {"questions": 100, "hints": 100}, "GPT-4-Vanilla-answer-aware": {"questions": 100, "hints": 100}, "LLaMA-3.1-8b-Vanilla-answer-aware": {"questions": 100, "hints": 100}, "LLaMA-3.1-405b-Vanilla-answer-agnostic": {"questions": 100, "hints": 100}, "LLaMA-3.1-8b-Vanilla-answer-agnostic": {"questions": 100, "hints": 100}, "LLaMA-3.1-70b-FTwA-answer-aware": {"questions": 100, "hints": 100}, "LLaMA-3.1-70b-FTwoA-answer-agnostic": {"questions": 100, "hints": 100}, "LLaMA-3.1-405b-Vanilla-answer-aware": {"questions": 100, "hints": 100}, "LLaMA-3.1-8b-FTwoA-answer-agnostic": {"questions": 100, "hints": 100}, "LLaMA-3.1-70b-Vanilla-answer-agnostic": {"questions": 100, "hints": 100}, "LLaMA-3.1-70b-Vanilla-answer-aware": {"questions": 100, "hints": 100}}}}
+{"hintqa": {"name": "HintQA", "version": "1.0", "description": "This dataset is generated using the method proposed in the TriviaHG paper for the test set questions of TriviaQA, NQ, and WebQ datasets. For each question, up to seven hints are generated and evaluated using various metrics.", "url": "https://aclanthology.org/2024.findings-emnlp.546/", "subsets": {"WebQ-Vanilla": {"questions": 2032, "hints": 15812}, "WebQ-Finetuned": {"questions": 2032, "hints": 16978}, "NQ-Vanilla": {"questions": 3610, "hints": 30976}, "NQ-Finetuned": {"questions": 3610, "hints": 33131}, "TriviaQA-Vanilla": {"questions": 11313, "hints": 103018}, "TriviaQA-Finetuned": {"questions": 11313, "hints": 105709}}}, "kg-hint": {"name": "KG-Hint", "version": "1.0", "description": "This dataset is generated using the AutomaticHintGeneration system. The system processes an .xlsx file containing the question, answer, and category. It retrieves information related to the answer from Wikidata and Wikipedia to generate hints. The answer must be a Wikipedia entity and can be a person, year, or location.", "url": "https://dl.acm.org/doi/10.1145/3578337.3605119", "subsets": {"entire": {"questions": 30, "hints": 307}}}, "triviahg": {"name": "TriviaHG", "version": "1.0", "description": "TriviaHG is an extensive dataset crafted specifically for hint generation in question answering.", "url": "https://dl.acm.org/doi/abs/10.1145/3626772.3657855", "subsets": {"LLaMA_13b_Vanilla": {"questions": 100, "hints": 896}, "WizardLM_70b": {"questions": 100, "hints": 941}, "GPT_4": {"questions": 100, "hints": 949}, "Copilot": {"questions": 100, "hints": 970}, "LLaMA_70b_Finetuned": {"questions": 100, "hints": 924}, "LLaMA_70b_Vanilla": {"questions": 100, "hints": 683}, "Gemini": {"questions": 100, "hints": 942}, "LLaMA_7b_Finetuned": {"questions": 100, "hints": 923}, "LLaMA_7b_Vanilla": {"questions": 100, "hints": 840}, "LLaMA_13b_Finetuned": {"questions": 100, "hints": 921}, "GPT_3.5": {"questions": 100, "hints": 898}, "test": {"questions": 1000, "hints": 9617}, "validation": {"questions": 1000, "hints": 9638}, "training": {"questions": 14645, "hints": 140973}}}, "wikihint": {"name": "WikiHint", "version": "1.0", "description": "The dataset was created using Amazon Mechanical Turk (MTurk). Workers were instructed to write five hints based on a Wikipedia article as their knowledge base. After writing all five hints, they ranked their hints by usefulness. A hint is ranked higher if it is more likely to lead the user to the correct answer. The best hint is assigned a rank of 1, and the worst hint is assigned a rank of 5.", "url": "https://github.com/FloGerhold/Hint_Dataset", "subsets": {"train": {"questions": 900, "hints": 4500}, "test": {"questions": 100, "hints": 500}, "LLaMA-3.1-8b-FTwA-answer-aware": {"questions": 100, "hints": 100}, "GPT-4-Vanilla-answer-agnostic": {"questions": 100, "hints": 100}, "GPT-4-Vanilla-answer-aware": {"questions": 100, "hints": 100}, "LLaMA-3.1-8b-Vanilla-answer-aware": {"questions": 100, "hints": 100}, "LLaMA-3.1-405b-Vanilla-answer-agnostic": {"questions": 100, "hints": 100}, "LLaMA-3.1-8b-Vanilla-answer-agnostic": {"questions": 100, "hints": 100}, "LLaMA-3.1-70b-FTwA-answer-aware": {"questions": 100, "hints": 100}, "LLaMA-3.1-70b-FTwoA-answer-agnostic": {"questions": 100, "hints": 100}, "LLaMA-3.1-405b-Vanilla-answer-aware": {"questions": 100, "hints": 100}, "LLaMA-3.1-8b-FTwoA-answer-agnostic": {"questions": 100, "hints": 100}, "LLaMA-3.1-70b-Vanilla-answer-agnostic": {"questions": 100, "hints": 100}, "LLaMA-3.1-70b-Vanilla-answer-aware": {"questions": 100, "hints": 100}}}}

datasets/triviahg.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:077ad5ac5a7e918184c7f48a917ecbbb71f41442976e15e877b683e6671d7652
-size 40273077
+oid sha256:25e3a330d9f38f9c52e102d3a4cb633905d4300492484c4892e490fc7958f702
+size 40273088

datasets/wikihint.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2fe7e79ed85206409ae11b9d95c32934af2d2a6d20752f8cb9cb6346860d9cd
+oid sha256:a67e0c1c52b4a6030ff3692dff8c6cb021fc74d0610395f419aa300338c341ba
 size 1931395