{ "cells": [ { "cell_type": "code", "execution_count": 2, "id": "d0191263", "metadata": {}, "outputs": [], "source": [ "#load in metadata.jsonl file to view it\n", "import json\n", "import jsonlines\n", "import pandas as pd\n", "import os\n", "import numpy as np\n", "\n", "# Load the metadata.jsonl file\n", "with jsonlines.open('metadata.jsonl') as reader:\n", " metadata = [obj for obj in reader]" ] }, { "cell_type": "code", "execution_count": 5, "id": "6820d40d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
task_idQuestionLevelFinal answerfile_nameAnnotator Metadata
0c61d22de-5f6c-4958-a7f6-5e9707bd3466A paper about AI regulation that was originall...2egalitarian{'Steps': '1. Go to arxiv.org and navigate to ...
117b5a6a3-bc87-42e8-b0fb-6ab0781ef2ccI’m researching species that became invasive a...234689{'Steps': '1. Search the web for “finding nemo...
204a04a9b-226c-43fd-b319-d5e89743676fIf we assume all articles published by Nature ...241{'Steps': '1. Find how many articles were publ...
314569e28-c88c-43e4-8c32-097d35b9a67dIn Unlambda, what exact charcter or text needs...2backtick{'Steps': '1. Searched \"Unlambda syntax\" onlin...
4e1fc63a2-da7a-432f-be78-7c4a95598703If Eliud Kipchoge could maintain his record-ma...117{'Steps': '1. Googled Eliud Kipchoge marathon ...
\n", "
" ], "text/plain": [ " task_id \\\n", "0 c61d22de-5f6c-4958-a7f6-5e9707bd3466 \n", "1 17b5a6a3-bc87-42e8-b0fb-6ab0781ef2cc \n", "2 04a04a9b-226c-43fd-b319-d5e89743676f \n", "3 14569e28-c88c-43e4-8c32-097d35b9a67d \n", "4 e1fc63a2-da7a-432f-be78-7c4a95598703 \n", "\n", " Question Level Final answer \\\n", "0 A paper about AI regulation that was originall... 2 egalitarian \n", "1 I’m researching species that became invasive a... 2 34689 \n", "2 If we assume all articles published by Nature ... 2 41 \n", "3 In Unlambda, what exact charcter or text needs... 2 backtick \n", "4 If Eliud Kipchoge could maintain his record-ma... 1 17 \n", "\n", " file_name Annotator Metadata \n", "0 {'Steps': '1. Go to arxiv.org and navigate to ... \n", "1 {'Steps': '1. Search the web for “finding nemo... \n", "2 {'Steps': '1. Find how many articles were publ... \n", "3 {'Steps': '1. Searched \"Unlambda syntax\" onlin... \n", "4 {'Steps': '1. Googled Eliud Kipchoge marathon ... " ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Convert the list of dictionaries to a DataFrame\n", "df = pd.DataFrame(metadata)\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 54, "id": "9b7659b2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'Steps': '1. Search the web for “finding nemo main character”.\\n2. Note the results, which state that the main character is a clownfish.\\n3. Search the web for “usgs nonnative species database”.\\n4. Click result for the Nonindigenous Aquatic Species site.\\n5. Click “Marine Fishes”.\\n6. Click “Species List of Nonindigenous Marine Fish”.\\n7. Scroll through the list until I find the clown anenomefish, and click “Collection info”.\\n8. Note the place that a clown anenomefish was found, in Fred Howard Park at the Gulf of Mexico.\\n9. Search the web for “fred howard park florida zip code”.\\n10. Note the zip code, 34689. Since only one clownfish was found before the year 2020, this is the answer.',\n", " 'Number of steps': '10',\n", " 'How long did this take?': '5 minutes',\n", " 'Tools': '1. Search engine\\n2. Web browser',\n", " 'Number of tools': '2'}" ] }, "execution_count": 54, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#convert metadata into dataframe and view the annotator column\n", "df = pd.DataFrame(metadata)\n", "df.head()\n", "df['Annotator Metadata'][1]" ] }, { "cell_type": "code", "execution_count": 65, "id": "73c7bbaa", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "In July 2, 1959 United States standards for grades of processed fruits,\n", "vegetables, and certain other products listed as dehydrated, consider the items\n", "in the \"dried and dehydrated section\" specifically marked as dehydrated along\n", "with any items in the Frozen/Chilled section that contain the whole name of the\n", "item, but not if they're marked Chilled. As of August 2023, what is the\n", "percentage (to the nearest percent) of those standards that have been superseded\n", "by a new version since the date given in the 1959 standards?\n" ] } ], "source": [ "#print the question and answer for the 11th row, print it out in wrapped text.\n", "import textwrap\n", "print(textwrap.fill(df['Question'][10], width=80))" ] }, { "cell_type": "code", "execution_count": 62, "id": "f30fb061", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'Steps': '1. Search the web for \"PDB ID 5wb7\"\\n2. Navigate to https://www.rcsb.org/structure/5wb7 from the search results page\\n3. Download the PDB file from the landing page.\\n4. Process the PDB file using Python and Biopython to calculate the distance between the first two atoms listed in the file. (1.4564234018325806 Å)\\nfrom Bio.PDB import PDBParser\\nparser = PDBParser()\\nstructure = parser.get_structure(\"5wb7\", \"5wb7.pdb\")\\nfor atom in structure.get_atoms():\\n atom1 = atom\\n break\\nfor atom in structure.get_atoms():\\n if atom != atom1:\\n atom2 = atom\\n break\\ndistance = atom1 - atom2\\nprint(f\"{distance}\")\\n5. Round the result to the nearest picometer (1.456)',\n", " 'Number of steps': '5',\n", " 'How long did this take?': '45 minutes',\n", " 'Tools': '1. Web browser\\n2. Search engine\\n3. File handling\\n4. Python\\n5. Calculator ',\n", " 'Number of tools': '5'}" ] }, "execution_count": 62, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df['Annotator Metadata'][11]" ] }, { "cell_type": "code", "execution_count": null, "id": "54bf634a", "metadata": {}, "outputs": [], "source": [ "# Extract all tools from Annotator Metadata\n", "all_tools = []\n", "for entry in metadata:\n", " tools = entry['Annotator Metadata']['Tools'].split('\\n')\n", " # Remove numbering and leading/trailing whitespace from each tool\n", " tools = [t.split('. ')[-1].strip() for t in tools]\n", " all_tools.extend(tools)\n", "\n", "# Count unique tools\n", "tool_counts = pd.Series(all_tools).value_counts()\n", "#print out the top 10 tools with counts greater than 5 as a DataFrame\n", "tool_counts_df = tool_counts.reset_index()\n", "tool_counts_df.columns = ['Tool', 'Count']\n", "tool_counts_df = tool_counts_df.sort_values(by='Count', ascending=False)\n", "# print out unique tool names\n", "# Save the tool counts to a CSV file\n", "tool_counts_df.to_csv('tool_counts.csv', index=False)" ] }, { "cell_type": "code", "execution_count": 37, "id": "1f0a65e7", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['Web browser',\n", " 'Search engine',\n", " 'Calculator',\n", " 'Image recognition tools',\n", " 'search engine',\n", " 'None',\n", " 'web browser',\n", " 'A search engine',\n", " 'A web browser',\n", " 'PDF viewer',\n", " 'PDF access']" ] }, "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tool_counts_df.Tool[tool_counts_df.Count > 5].values.tolist()" ] }, { "cell_type": "code", "execution_count": 57, "id": "a6f475af", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ToolCount
0Web browser95
1Search engine88
2Calculator30
3Image recognition tools11
4search engine9
.........
88Wikipedia1
89Video capability1
90Image processing tools1
91Image recognition software1
92YouTube1
\n", "

93 rows × 2 columns

\n", "
" ], "text/plain": [ " Tool Count\n", "0 Web browser 95\n", "1 Search engine 88\n", "2 Calculator 30\n", "3 Image recognition tools 11\n", "4 search engine 9\n", ".. ... ...\n", "88 Wikipedia 1\n", "89 Video capability 1\n", "90 Image processing tools 1\n", "91 Image recognition software 1\n", "92 YouTube 1\n", "\n", "[93 rows x 2 columns]" ] }, "execution_count": 57, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tool_counts_df" ] } ], "metadata": { "kernelspec": { "display_name": "hf_agent", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.17" } }, "nbformat": 4, "nbformat_minor": 5 }