Delete exploration.ipynb

exploration.ipynb

@@ -1,190 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "94bd79f6",
-   "metadata": {},
-   "source": [
-    "# Overview of the GAIA dataset"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "773d3352",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'task_id': 'c61d22de-5f6c-4958-a7f6-5e9707bd3466',\n",
-       " 'Question': 'A paper about AI regulation that was originally submitted to arXiv.org in June 2022 shows a figure with three axes, where each axis has a label word at both ends. Which of these words is used to describe a type of society in a Physics and Society article submitted to arXiv.org on August 11, 2016?',\n",
-       " 'Level': 2,\n",
-       " 'Final answer': 'egalitarian',\n",
-       " 'file_name': '',\n",
-       " 'Annotator Metadata': {'Steps': '1. Go to arxiv.org and navigate to the Advanced Search page.\\n2. Enter \"AI regulation\" in the search box and select \"All fields\" from the dropdown.\\n3. Enter 2022-06-01 and 2022-07-01 into the date inputs, select \"Submission date (original)\", and submit the search.\\n4. Go through the search results to find the article that has a figure with three axes and labels on each end of the axes, titled \"Fairness in Agreement With European Values: An Interdisciplinary Perspective on AI Regulation\".\\n5. Note the six words used as labels: deontological, egalitarian, localized, standardized, utilitarian, and consequential.\\n6. Go back to arxiv.org\\n7. Find \"Physics and Society\" and go to the page for the \"Physics and Society\" category.\\n8. Note that the tag for this category is \"physics.soc-ph\".\\n9. Go to the Advanced Search page.\\n10. Enter \"physics.soc-ph\" in the search box and select \"All fields\" from the dropdown.\\n11. Enter 2016-08-11 and 2016-08-12 into the date inputs, select \"Submission date (original)\", and submit the search.\\n12. Search for instances of the six words in the results to find the paper titled \"Phase transition from egalitarian to hierarchical societies driven by competition between cognitive and social constraints\", indicating that \"egalitarian\" is the correct answer.',\n",
-       "  'Number of steps': '12',\n",
-       "  'How long did this take?': '8 minutes',\n",
-       "  'Tools': '1. Web browser\\n2. Image recognition tools (to identify and parse a figure with three axes)',\n",
-       "  'Number of tools': '2'}}"
-      ]
-     },
-     "execution_count": 1,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "\n",
-    "import json\n",
-    "# Load the metadata.jsonl file\n",
-    "with open('metadata.jsonl', 'r') as jsonl_file:\n",
-    "    json_list = list(jsonl_file)\n",
-    "\n",
-    "json_QA = []\n",
-    "for json_str in json_list:\n",
-    "    json_data = json.loads(json_str)\n",
-    "    json_QA.append(json_data)\n",
-    "    \n",
-    "\n",
-    "json_QA[0]\n",
-    "    "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "be320045",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "==================================================\n",
-      "Task ID: 0ff53813-3367-4f43-bcbd-3fd725c1bf4b\n",
-      "Question: What two-word type of model did Manash Pratim Kashyap's and PS Fader's studies in customer retention studies published during 2018-2019 have in common (no punctuation)?\n",
-      "Level: 2\n",
-      "Final Answer: beta geometric\n",
-      "Annotator Metadata: \n",
-      "  ├── Steps: \n",
-      "  │      ├── 1. Searched \"Manash Pratim Kashyap customer retention\" on Google.\n",
-      "  │      ├── 2. Opened https://www.journalijar.com/article/26843/a-simple-model-for-analyzing-the-customer-retention-comparing-rural-and-urban-store/.\n",
-      "  │      ├── 3. Noted \"discrete time beta geometric model\" in the abstract.\n",
-      "  │      ├── 4. Searched \"PS Fader customer retention\" on Google.\n",
-      "  │      ├── 5. Opened https://www.sciencedirect.com/science/article/abs/pii/S1094996807700233.\n",
-      "  │      ├── 6. Noted \"basic model (known as a “shifted-beta-geometric”)\" in the abstract.\n",
-      "  │      ├── 7. Extracted the two words in common.\n",
-      "  ├── Number of steps: 6\n",
-      "  ├── How long did this take?: 10 minutes\n",
-      "  ├── Tools:\n",
-      "  │      ├── 1. Web browser\n",
-      "  │      ├── 2. Search engine\n",
-      "  └── Number of tools: 2\n",
-      "==================================================\n"
-     ]
-    }
-   ],
-   "source": [
-    "\n",
-    "import random\n",
-    "# random.seed(42)\n",
-    "random_samples = random.sample(json_QA, 1)\n",
-    "for sample in random_samples:\n",
-    "    print(\"=\" * 50)\n",
-    "    print(f\"Task ID: {sample['task_id']}\")\n",
-    "    print(f\"Question: {sample['Question']}\")\n",
-    "    print(f\"Level: {sample['Level']}\")\n",
-    "    print(f\"Final Answer: {sample['Final answer']}\")\n",
-    "    print(f\"Annotator Metadata: \")\n",
-    "    print(f\"  ├── Steps: \")\n",
-    "    for step in sample['Annotator Metadata']['Steps'].split('\\n'):\n",
-    "        print(f\"  │      ├── {step}\")\n",
-    "    print(f\"  ├── Number of steps: {sample['Annotator Metadata']['Number of steps']}\")\n",
-    "    print(f\"  ├── How long did this take?: {sample['Annotator Metadata']['How long did this take?']}\")\n",
-    "    print(f\"  ├── Tools:\")\n",
-    "    for tool in sample['Annotator Metadata']['Tools'].split('\\n'):\n",
-    "        print(f\"  │      ├── {tool}\")\n",
-    "    print(f\"  └── Number of tools: {sample['Annotator Metadata']['Number of tools']}\")\n",
-    "print(\"=\" * 50)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "64c5ca54",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[{'task_id': 'c61d22de-5f6c-4958-a7f6-5e9707bd3466',\n",
-       "  'Question': 'A paper about AI regulation that was originally submitted to arXiv.org in June 2022 shows a figure with three axes, where each axis has a label word at both ends. Which of these words is used to describe a type of society in a Physics and Society article submitted to arXiv.org on August 11, 2016?',\n",
-       "  'Level': 2,\n",
-       "  'Final answer': 'egalitarian',\n",
-       "  'file_name': '',\n",
-       "  'Annotator Metadata': {'Steps': '1. Go to arxiv.org and navigate to the Advanced Search page.\\n2. Enter \"AI regulation\" in the search box and select \"All fields\" from the dropdown.\\n3. Enter 2022-06-01 and 2022-07-01 into the date inputs, select \"Submission date (original)\", and submit the search.\\n4. Go through the search results to find the article that has a figure with three axes and labels on each end of the axes, titled \"Fairness in Agreement With European Values: An Interdisciplinary Perspective on AI Regulation\".\\n5. Note the six words used as labels: deontological, egalitarian, localized, standardized, utilitarian, and consequential.\\n6. Go back to arxiv.org\\n7. Find \"Physics and Society\" and go to the page for the \"Physics and Society\" category.\\n8. Note that the tag for this category is \"physics.soc-ph\".\\n9. Go to the Advanced Search page.\\n10. Enter \"physics.soc-ph\" in the search box and select \"All fields\" from the dropdown.\\n11. Enter 2016-08-11 and 2016-08-12 into the date inputs, select \"Submission date (original)\", and submit the search.\\n12. Search for instances of the six words in the results to find the paper titled \"Phase transition from egalitarian to hierarchical societies driven by competition between cognitive and social constraints\", indicating that \"egalitarian\" is the correct answer.',\n",
-       "   'Number of steps': '12',\n",
-       "   'How long did this take?': '8 minutes',\n",
-       "   'Tools': '1. Web browser\\n2. Image recognition tools (to identify and parse a figure with three axes)',\n",
-       "   'Number of tools': '2'}},\n",
-       " {'task_id': '17b5a6a3-bc87-42e8-b0fb-6ab0781ef2cc',\n",
-       "  'Question': 'I’m researching species that became invasive after people who kept them as pets released them. There’s a certain species of fish that was popularized as a pet by being the main character of the movie Finding Nemo. According to the USGS, where was this fish found as a nonnative species, before the year 2020? I need the answer formatted as the five-digit zip codes of the places the species was found, separated by commas if there is more than one place.',\n",
-       "  'Level': 2,\n",
-       "  'Final answer': '34689',\n",
-       "  'file_name': '',\n",
-       "  'Annotator Metadata': {'Steps': '1. Search the web for “finding nemo main character”.\\n2. Note the results, which state that the main character is a clownfish.\\n3. Search the web for “usgs nonnative species database”.\\n4. Click result for the Nonindigenous Aquatic Species site.\\n5. Click “Marine Fishes”.\\n6. Click “Species List of Nonindigenous Marine Fish”.\\n7. Scroll through the list until I find the clown anenomefish, and click “Collection info”.\\n8. Note the place that a clown anenomefish was found, in Fred Howard Park at the Gulf of Mexico.\\n9. Search the web for “fred howard park florida zip code”.\\n10. Note the zip code, 34689. Since only one clownfish was found before the year 2020, this is the answer.',\n",
-       "   'Number of steps': '10',\n",
-       "   'How long did this take?': '5 minutes',\n",
-       "   'Tools': '1. Search engine\\n2. Web browser',\n",
-       "   'Number of tools': '2'}},\n",
-       " {'task_id': '04a04a9b-226c-43fd-b319-d5e89743676f',\n",
-       "  'Question': 'If we assume all articles published by Nature in 2020 (articles, only, not book reviews/columns, etc) relied on statistical significance to justify their findings and they on average came to a p-value of 0.04, how many papers would be incorrect as to their claims of statistical significance? Round the value up to the next integer.',\n",
-       "  'Level': 2,\n",
-       "  'Final answer': '41',\n",
-       "  'file_name': '',\n",
-       "  'Annotator Metadata': {'Steps': '1. Find how many articles were published in Nature in 2020 by Googling \"articles submitted to nature 2020\"\\n2. Click through to Nature\\'s archive for 2020 and filter the results to only provide articles, not other types of publications: 1002\\n3. Find 4% of 1002 and round up: 40.08 > 41',\n",
-       "   'Number of steps': '3',\n",
-       "   'How long did this take?': '5 minutes',\n",
-       "   'Tools': '1. search engine\\n2. calculator',\n",
-       "   'Number of tools': '2'}}]"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "json_QA[0:3]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "d4ddf21d",
-   "metadata": {},
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.12"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}