{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import os\n", "os.chdir(\"../..\")" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import os.path as osp\n", "from pathlib import Path\n", "import numpy as np\n", "import pandas as pd\n", "import json" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "def _add_prefix_suffix_to_path(path: str, prefix: str, suffix: str) -> str:\n", " base_dir, filename = os.path.split(path)\n", " return os.path.join(base_dir, prefix + filename + suffix)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "pred_json_dict = {\n", " \"sca\": \"amlt/exp-verb-metric/infer-train-sca-ablat-lsj-scale_lr-110423-110723.running-2/last-fp16-ollm3bv2-large-lsj-1xlr.110423.octo-4x8-v100-16g-no_pre/vg-densecap-region_descriptions/infer/infer-visual_genome-region_descriptions_v1.2.0-test.json\",\n", " \"blip2-opt-2.7b\": \"amlt/exp-verb-metric/infer-sam_captioner/Salesforce/blip2-opt-2.7b/infer-post_processed/infer-visual_genome-densecap-local-densecap-test.json.post.json\",\n", " \"blip2-opt-2.7b-coco\": \"amlt/exp-verb-metric/infer-sam_captioner/Salesforce/blip2-opt-2.7b-coco/infer-post_processed/infer-visual_genome-densecap-local-densecap-test.json.post.json\",\n", " \"blip2-opt-6.7b\": \"amlt/exp-verb-metric/infer-sam_captioner/Salesforce/blip2-opt-6.7b/infer-post_processed/infer-visual_genome-densecap-local-densecap-test.json.post.json\",\n", " \"blip2-opt-6.7b-coco\": \"amlt/exp-verb-metric/infer-sam_captioner/Salesforce/blip2-opt-6.7b-coco/infer-post_processed/infer-visual_genome-densecap-local-densecap-test.json.post.json\",\n", "}" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "SCORE_PREFIXS = [\"noun_distance\", \"noun_recall\", \"verb_distance\", \"verb_recall\"]\n", "score_json_dict = {}\n", "for score_prefix in SCORE_PREFIXS:\n", " score_json_dict[score_prefix] = {}\n", " for pred_name, pred_json_path in pred_json_dict.items():\n", " score_json_dict[score_prefix][pred_name] = _add_prefix_suffix_to_path(pred_json_path, score_prefix + \"-\", \".json\")" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "# check the existence of the files\n", "for k, v in pred_json_dict.items():\n", " assert osp.exists(v), f\"{v} does not exist\"\n", "\n", "for score_prefix, _score_json_dict in score_json_dict.items():\n", " for pred_name, pred_json_path in _score_json_dict.items():\n", " assert osp.exists(pred_json_path), f\"{pred_json_path} does not exist\"" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
_idrefprednoun_distancenoun_recallverb_distanceverb_recall
00button on the mans jacketa button on a jacket0.6907870.6666670.00.0
11photography website for the company that took ...the name of the photographer0.5216510.0000000.00.0
22photographer logo and namethe writing is in white0.2346700.0000000.00.0
33black tie the man is wearingThe man is wearing a tie.1.0000001.0000001.01.0
44a vest that is under the coatThe man is wearing a black vest.0.7434700.5000000.00.0
\n", "
" ], "text/plain": [ " _id ref \\\n", "0 0 button on the mans jacket \n", "1 1 photography website for the company that took ... \n", "2 2 photographer logo and name \n", "3 3 black tie the man is wearing \n", "4 4 a vest that is under the coat \n", "\n", " pred noun_distance noun_recall \\\n", "0 a button on a jacket 0.690787 0.666667 \n", "1 the name of the photographer 0.521651 0.000000 \n", "2 the writing is in white 0.234670 0.000000 \n", "3 The man is wearing a tie. 1.000000 1.000000 \n", "4 The man is wearing a black vest. 0.743470 0.500000 \n", "\n", " verb_distance verb_recall \n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 1.0 1.0 \n", "4 0.0 0.0 " ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# load json and score and combine them into one dataframe\n", "\n", "def read_pred_score(pred_name):\n", " pred_json_path = pred_json_dict[pred_name]\n", " with open(pred_json_path, \"r\") as pred_json_path:\n", " pred_json = json.load(pred_json_path)\n", "\n", " scores = []\n", " for score_prefix in SCORE_PREFIXS:\n", " score_json_path = score_json_dict[score_prefix][pred_name]\n", " with open(score_json_path, \"r\") as score_json_path:\n", " score_json = json.load(score_json_path)\n", " scores.append(np.array(score_json))\n", " scores = np.stack(scores, axis=-1) # (N, num_score_type)\n", "\n", " if len(pred_json) != len(scores):\n", " raise ValueError(f\"pred_json and score_json have different length: {len(pred_json)} vs {len(scores)}, in {pred_json_path} and {score_json_path}\")\n", "\n", " conbined_json = []\n", " for pred, score in zip(pred_json, scores):\n", " pred_dict = {\n", " \"_id\": pred[\"_id\"],\n", " \"ref\": pred[\"references\"][0],\n", " \"pred\": pred[\"candidates\"][0],\n", " }\n", " score_dict = {score_prefix: score[i] for i, score_prefix in enumerate(SCORE_PREFIXS)}\n", " conbine_dict = {**pred_dict, **score_dict}\n", " conbined_json.append(conbine_dict)\n", "\n", " conbined_df = pd.DataFrame(conbined_json)\n", " return conbined_df\n", "\n", "pred_name = \"sca\"\n", "conbined_df = read_pred_score(pred_name)\n", "conbined_df.head()" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [], "source": [ "conbined_dict = {}\n", "for pred_name in pred_json_dict.keys():\n", " conbined_dict[pred_name] = read_pred_score(pred_name)\n" ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [], "source": [ "combined_df = pd.concat([conbined_dict[df_name].add_prefix(df_name + \"-\") for df_name in pred_json_dict.keys()], axis=1)" ] }, { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
sca-_idsca-refsca-predsca-noun_distancesca-noun_recallsca-verb_distancesca-verb_recallblip2-opt-2.7b-_idblip2-opt-2.7b-refblip2-opt-2.7b-pred...blip2-opt-6.7b-noun_recallblip2-opt-6.7b-verb_distanceblip2-opt-6.7b-verb_recallblip2-opt-6.7b-coco-_idblip2-opt-6.7b-coco-refblip2-opt-6.7b-coco-predblip2-opt-6.7b-coco-noun_distanceblip2-opt-6.7b-coco-noun_recallblip2-opt-6.7b-coco-verb_distanceblip2-opt-6.7b-coco-verb_recall
00button on the mans jacketa button on a jacket0.6907870.6666670.00.00button on the mans jacketa black and white photo of a person in a dark ......0.00.00.00button on the mans jacketa blurry image of a person standing in front o...0.3147360.00.00.0
11photography website for the company that took ...the name of the photographer0.5216510.0000000.00.01photography website for the company that took ...a black and white image of a line of white lin......0.00.00.01photography website for the company that took ...a close up of a clock with a black background\\n0.4033070.00.00.0
22photographer logo and namethe writing is in white0.2346700.0000000.00.02photographer logo and namea man is standing in front of a blue background\\n...0.00.00.02photographer logo and namea close up of a man wearing a white shirt and ...0.2828830.00.00.0
33black tie the man is wearingThe man is wearing a tie.1.0000001.0000001.01.03black tie the man is wearinga man wearing a suit and tie\\n...0.51.01.03black tie the man is wearinga close up of a woman's legs in a pair of high...0.5914170.00.00.0
44a vest that is under the coatThe man is wearing a black vest.0.7434700.5000000.00.04a vest that is under the coata man is sitting on a chair\\n...0.00.00.04a vest that is under the coata close up of a pair of shoes with a person we...0.4601470.00.00.0
\n", "

5 rows × 35 columns

\n", "
" ], "text/plain": [ " sca-_id sca-ref \\\n", "0 0 button on the mans jacket \n", "1 1 photography website for the company that took ... \n", "2 2 photographer logo and name \n", "3 3 black tie the man is wearing \n", "4 4 a vest that is under the coat \n", "\n", " sca-pred sca-noun_distance sca-noun_recall \\\n", "0 a button on a jacket 0.690787 0.666667 \n", "1 the name of the photographer 0.521651 0.000000 \n", "2 the writing is in white 0.234670 0.000000 \n", "3 The man is wearing a tie. 1.000000 1.000000 \n", "4 The man is wearing a black vest. 0.743470 0.500000 \n", "\n", " sca-verb_distance sca-verb_recall blip2-opt-2.7b-_id \\\n", "0 0.0 0.0 0 \n", "1 0.0 0.0 1 \n", "2 0.0 0.0 2 \n", "3 1.0 1.0 3 \n", "4 0.0 0.0 4 \n", "\n", " blip2-opt-2.7b-ref \\\n", "0 button on the mans jacket \n", "1 photography website for the company that took ... \n", "2 photographer logo and name \n", "3 black tie the man is wearing \n", "4 a vest that is under the coat \n", "\n", " blip2-opt-2.7b-pred ... \\\n", "0 a black and white photo of a person in a dark ... ... \n", "1 a black and white image of a line of white lin... ... \n", "2 a man is standing in front of a blue background\\n ... \n", "3 a man wearing a suit and tie\\n ... \n", "4 a man is sitting on a chair\\n ... \n", "\n", " blip2-opt-6.7b-noun_recall blip2-opt-6.7b-verb_distance \\\n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 0.5 1.0 \n", "4 0.0 0.0 \n", "\n", " blip2-opt-6.7b-verb_recall blip2-opt-6.7b-coco-_id \\\n", "0 0.0 0 \n", "1 0.0 1 \n", "2 0.0 2 \n", "3 1.0 3 \n", "4 0.0 4 \n", "\n", " blip2-opt-6.7b-coco-ref \\\n", "0 button on the mans jacket \n", "1 photography website for the company that took ... \n", "2 photographer logo and name \n", "3 black tie the man is wearing \n", "4 a vest that is under the coat \n", "\n", " blip2-opt-6.7b-coco-pred \\\n", "0 a blurry image of a person standing in front o... \n", "1 a close up of a clock with a black background\\n \n", "2 a close up of a man wearing a white shirt and ... \n", "3 a close up of a woman's legs in a pair of high... \n", "4 a close up of a pair of shoes with a person we... \n", "\n", " blip2-opt-6.7b-coco-noun_distance blip2-opt-6.7b-coco-noun_recall \\\n", "0 0.314736 0.0 \n", "1 0.403307 0.0 \n", "2 0.282883 0.0 \n", "3 0.591417 0.0 \n", "4 0.460147 0.0 \n", "\n", " blip2-opt-6.7b-coco-verb_distance blip2-opt-6.7b-coco-verb_recall \n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 0.0 0.0 \n", "4 0.0 0.0 \n", "\n", "[5 rows x 35 columns]" ] }, "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ "combined_df.head()" ] }, { "cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
sca-refsca-predsca-verb_distancesca-verb_recallblip2-opt-2.7b-refblip2-opt-2.7b-predblip2-opt-2.7b-verb_distanceblip2-opt-2.7b-verb_recallblip2-opt-2.7b-coco-refblip2-opt-2.7b-coco-predblip2-opt-2.7b-coco-verb_distanceblip2-opt-2.7b-coco-verb_recallblip2-opt-6.7b-refblip2-opt-6.7b-predblip2-opt-6.7b-verb_distanceblip2-opt-6.7b-verb_recallblip2-opt-6.7b-coco-refblip2-opt-6.7b-coco-predblip2-opt-6.7b-coco-verb_distanceblip2-opt-6.7b-coco-verb_recall
0button on the mans jacketa button on a jacket0.00.0button on the mans jacketa black and white photo of a person in a dark ...0.00.0button on the mans jacketa blurry picture of a black and white picture ...0.00.0button on the mans jacketa black and white photo of a person with a whi...0.00.0button on the mans jacketa blurry image of a person standing in front o...0.00.0
1photography website for the company that took ...the name of the photographer0.00.0photography website for the company that took ...a black and white image of a line of white lin...0.00.0photography website for the company that took ...a blurry image of a bunch of white lines on a ...0.00.0photography website for the company that took ...a black and white image of a wave pattern\\n0.00.0photography website for the company that took ...a close up of a clock with a black background\\n0.00.0
2photographer logo and namethe writing is in white0.00.0photographer logo and namea man is standing in front of a blue background\\n0.00.0photographer logo and namea man is holding a camera and taking a picture\\n0.00.0photographer logo and nameprofile picture for sc photography\\n0.00.0photographer logo and namea close up of a man wearing a white shirt and ...0.00.0
3black tie the man is wearingThe man is wearing a tie.1.01.0black tie the man is wearinga man wearing a suit and tie\\n1.01.0black tie the man is wearinga man wearing a tie and a shirt with a tie\\n1.01.0black tie the man is wearinga man wearing a suit and tie\\n1.01.0black tie the man is wearinga close up of a woman's legs in a pair of high...0.00.0
4a vest that is under the coatThe man is wearing a black vest.0.00.0a vest that is under the coata man is sitting on a chair\\n0.00.0a vest that is under the coata black cat sitting on a couch with a blanket ...0.00.0a vest that is under the coata man is sitting on a chair\\n0.00.0a vest that is under the coata close up of a pair of shoes with a person we...0.00.0
\n", "
" ], "text/plain": [ " sca-ref \\\n", "0 button on the mans jacket \n", "1 photography website for the company that took ... \n", "2 photographer logo and name \n", "3 black tie the man is wearing \n", "4 a vest that is under the coat \n", "\n", " sca-pred sca-verb_distance sca-verb_recall \\\n", "0 a button on a jacket 0.0 0.0 \n", "1 the name of the photographer 0.0 0.0 \n", "2 the writing is in white 0.0 0.0 \n", "3 The man is wearing a tie. 1.0 1.0 \n", "4 The man is wearing a black vest. 0.0 0.0 \n", "\n", " blip2-opt-2.7b-ref \\\n", "0 button on the mans jacket \n", "1 photography website for the company that took ... \n", "2 photographer logo and name \n", "3 black tie the man is wearing \n", "4 a vest that is under the coat \n", "\n", " blip2-opt-2.7b-pred \\\n", "0 a black and white photo of a person in a dark ... \n", "1 a black and white image of a line of white lin... \n", "2 a man is standing in front of a blue background\\n \n", "3 a man wearing a suit and tie\\n \n", "4 a man is sitting on a chair\\n \n", "\n", " blip2-opt-2.7b-verb_distance blip2-opt-2.7b-verb_recall \\\n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 1.0 1.0 \n", "4 0.0 0.0 \n", "\n", " blip2-opt-2.7b-coco-ref \\\n", "0 button on the mans jacket \n", "1 photography website for the company that took ... \n", "2 photographer logo and name \n", "3 black tie the man is wearing \n", "4 a vest that is under the coat \n", "\n", " blip2-opt-2.7b-coco-pred \\\n", "0 a blurry picture of a black and white picture ... \n", "1 a blurry image of a bunch of white lines on a ... \n", "2 a man is holding a camera and taking a picture\\n \n", "3 a man wearing a tie and a shirt with a tie\\n \n", "4 a black cat sitting on a couch with a blanket ... \n", "\n", " blip2-opt-2.7b-coco-verb_distance blip2-opt-2.7b-coco-verb_recall \\\n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 1.0 1.0 \n", "4 0.0 0.0 \n", "\n", " blip2-opt-6.7b-ref \\\n", "0 button on the mans jacket \n", "1 photography website for the company that took ... \n", "2 photographer logo and name \n", "3 black tie the man is wearing \n", "4 a vest that is under the coat \n", "\n", " blip2-opt-6.7b-pred \\\n", "0 a black and white photo of a person with a whi... \n", "1 a black and white image of a wave pattern\\n \n", "2 profile picture for sc photography\\n \n", "3 a man wearing a suit and tie\\n \n", "4 a man is sitting on a chair\\n \n", "\n", " blip2-opt-6.7b-verb_distance blip2-opt-6.7b-verb_recall \\\n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 1.0 1.0 \n", "4 0.0 0.0 \n", "\n", " blip2-opt-6.7b-coco-ref \\\n", "0 button on the mans jacket \n", "1 photography website for the company that took ... \n", "2 photographer logo and name \n", "3 black tie the man is wearing \n", "4 a vest that is under the coat \n", "\n", " blip2-opt-6.7b-coco-pred \\\n", "0 a blurry image of a person standing in front o... \n", "1 a close up of a clock with a black background\\n \n", "2 a close up of a man wearing a white shirt and ... \n", "3 a close up of a woman's legs in a pair of high... \n", "4 a close up of a pair of shoes with a person we... \n", "\n", " blip2-opt-6.7b-coco-verb_distance blip2-opt-6.7b-coco-verb_recall \n", "0 0.0 0.0 \n", "1 0.0 0.0 \n", "2 0.0 0.0 \n", "3 0.0 0.0 \n", "4 0.0 0.0 " ] }, "execution_count": 63, "metadata": {}, "output_type": "execute_result" } ], "source": [ "filtered_combined_df = combined_df[[i for i in combined_df.columns if \"verb\" in i or \"pred\" in i or \"ref\" in i]]\n", "filtered_combined_df.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 67, "metadata": {}, "outputs": [], "source": [ "df = filtered_combined_df" ] }, { "cell_type": "code", "execution_count": 72, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
sca-refsca-predsca-verb_distancesca-verb_recallblip2-opt-2.7b-refblip2-opt-2.7b-predblip2-opt-2.7b-verb_distanceblip2-opt-2.7b-verb_recallblip2-opt-2.7b-coco-refblip2-opt-2.7b-coco-predblip2-opt-2.7b-coco-verb_distanceblip2-opt-2.7b-coco-verb_recallblip2-opt-6.7b-refblip2-opt-6.7b-predblip2-opt-6.7b-verb_distanceblip2-opt-6.7b-verb_recallblip2-opt-6.7b-coco-refblip2-opt-6.7b-coco-predblip2-opt-6.7b-coco-verb_distanceblip2-opt-6.7b-coco-verb_recall
12766truck hauling an airplane's cargoA truck is next to the plane.0.0000000.0truck hauling an airplane's cargoa truck is driving down the road\\n0.5998860.0truck hauling an airplane's cargoa truck is carrying boxes on the back of it\\n0.7265370.0truck hauling an airplane's cargoa truck with a load of boxes on the back\\n0.0000000.0truck hauling an airplane's cargoa truck with a load of furniture on the back o...0.0000000.0
238035sliding glass doorthe window is open0.0000000.0sliding glass doora man is holding a white towel\\n0.5996720.0sliding glass doora blurry picture of a cell phone with a red li...0.0000000.0sliding glass doora person is holding a cell phone in front of a...0.5996720.0sliding glass doora blurry picture of a man standing in front of...0.6124570.0
13302Tree trunk providing shadethe trunk of a tree0.0000000.0Tree trunk providing shadea man is standing in front of a building\\n0.5996510.0Tree trunk providing shadea blurry image of a giraffe standing in a field\\n0.5996510.0Tree trunk providing shadea person is standing in front of a building\\n0.5996510.0Tree trunk providing shadea man standing in the shade of a tree with a s...0.5996510.0
123060railing is greena green fence on the side of the platform0.0000000.0railing is greena train is pulling into a station\\n0.5996230.0railing is greena man is walking down a sidewalk with a skateb...0.4956650.0railing is greena train is coming\\n0.4180800.0railing is greena man walking down a sidewalk next to a green ...0.4956650.0
96716person siting on the sidea man standing in front of a wall0.7427290.0person siting on the sidea man is painting a wall\\n0.5993270.0person siting on the sidea woman is sitting on a chair in a bathroom\\n0.6562250.0person siting on the sidea man standing in front of a large white board\\n0.7427290.0person siting on the sidea woman standing in front of a large white boa...0.7427290.0
123993man eating chocolate glazed doughnut with spri...a man eating a donut0.5993210.5man eating chocolate glazed doughnut with spri...a man eating a doughnut\\n0.5993210.5man eating chocolate glazed doughnut with spri...a man eating a doughnut with sprinkles on it\\n0.5993210.5man eating chocolate glazed doughnut with spri...a man eating a donut\\n0.5993210.5man eating chocolate glazed doughnut with spri...a man with glasses eating a doughnut with spri...0.5993210.5
108952Catcher crouching down in the dirtthe catcher is squatting0.7137590.0Catcher crouching down in the dirta baseball player is throwing a ball\\n0.5992970.0Catcher crouching down in the dirta blurry picture of a table with a bunch of st...0.0000000.0Catcher crouching down in the dirta baseball player is swinging a bat\\n0.6489130.0Catcher crouching down in the dirta blurry image of a baseball player in a unifo...0.5637070.0
188897ladels and whisk hanging from ceilingutensils hanging from the ceiling0.5991890.5ladels and whisk hanging from ceilinga kitchen with a mixer and a whisk hanging fro...0.5991890.5ladels and whisk hanging from ceilinga close up of a mixer with a whisk and a bowl\\n0.0000000.0ladels and whisk hanging from ceilinga kitchen with a stove and a mixer\\n0.0000000.0ladels and whisk hanging from ceilinga ceiling fan and a metal whisk hanging from a...0.5991890.5
139590a cutting boarding leaning against a walla wooden cutting board0.0000000.0a cutting boarding leaning against a walla wooden cutting board sitting on top of a sink\\n0.5991700.0a cutting boarding leaning against a walla man is sitting on a chair with a book in fro...0.5991700.0a cutting boarding leaning against a walla sink with a faucet and a wooden cutting board\\n0.0000000.0a cutting boarding leaning against a walla close up of a sink with a faucet and a towel\\n0.0000000.0
184970Guy wearing white shirt and tiea man wearing a white shirt0.5990680.5Guy wearing white shirt and tiea man wearing a white shirt\\n0.5990680.5Guy wearing white shirt and tiea man in a white shirt and tie standing next t...0.7572760.5Guy wearing white shirt and tiea man wearing a white shirt\\n0.5990680.5Guy wearing white shirt and tiea man in a white shirt and tie standing outside\\n0.7572760.5
114386The brown and white bow tie the girl is wearing.the bow tie is black and white0.0000000.0The brown and white bow tie the girl is wearing.a man wearing a bow tie\\n0.5990680.5The brown and white bow tie the girl is wearing.a close up of a man wearing a white shirt and ...0.5990680.5The brown and white bow tie the girl is wearing.a man wearing a bow tie\\n0.5990680.5The brown and white bow tie the girl is wearing.a person wearing a brown bow tie with a white ...0.5990680.5
49809the man is wearing a vest and tiea man wearing a tie0.5990680.5the man is wearing a vest and tieman wearing a black vest\\n0.5990680.5the man is wearing a vest and tiea man in a suit and tie taking a selfie\\n0.7346030.5the man is wearing a vest and tiea man wearing a vest\\n0.5990680.5the man is wearing a vest and tiea man in a vest and tie taking a selfie\\n0.7346030.5
89413Man is wearing a suit and tiea man wearing a suit0.5990680.5Man is wearing a suit and tieman wearing a suit\\n0.5990680.5Man is wearing a suit and tiea man in a suit and tie taking a selfie\\n0.7346030.5Man is wearing a suit and tieman wearing a suit\\n0.5990680.5Man is wearing a suit and tiea man in a suit taking a selfie in a mirror\\n0.2727460.0
169682A man is wearing a vest and tiea man wearing a vest0.5990680.5A man is wearing a vest and tiea man wearing a vest\\n0.5990680.5A man is wearing a vest and tiea man in a sweater vest and tie sitting at a t...0.7769050.5A man is wearing a vest and tiea man wearing a vest\\n0.5990680.5A man is wearing a vest and tiea man in a sweater and tie sitting at a table\\n0.7769050.5
231217man wearing suit and tiea man wearing a suit0.5990680.5man wearing suit and tiea man wearing a tie\\n0.5990680.5man wearing suit and tiea man in a suit and tie sitting at a table wit...0.7769050.5man wearing suit and tiea man wearing a suit and tie\\n1.0000001.0man wearing suit and tiea man in a suit and tie sitting at a table\\n0.7769050.5
159392person wearing an orange and red plaid shirt a...a man wearing a red and white striped shirt0.5990680.5person wearing an orange and red plaid shirt a...a man wearing a tie\\n0.5990680.5person wearing an orange and red plaid shirt a...a young man in a tie and plaid shirt holding a...0.3521330.0person wearing an orange and red plaid shirt a...a man wearing a tie\\n0.5990680.5person wearing an orange and red plaid shirt a...a man in a tie and plaid shirt standing next t...0.2882160.0
137457a person sitting at a table to eata balding man in a plaid shirt0.2162950.0a person sitting at a table to eata group of people sitting at a table\\n0.5989280.5a person sitting at a table to eata group of people sitting at a table with a pl...0.5989280.5a person sitting at a table to eata group of people sitting at a table\\n0.5989280.5a person sitting at a table to eata group of people sitting at a table with food...0.5989280.5
137455a person sitting at a table to eata woman wearing a gray shirt0.3460790.0a person sitting at a table to eata group of people sitting at a table\\n0.5989280.5a person sitting at a table to eata group of people sitting at a table with plat...0.5989280.5a person sitting at a table to eata woman wearing a green sweater\\n0.3460790.0a person sitting at a table to eata group of people sitting at a table eating fo...0.7399330.5
137462a person sitting at a table to eata man wearing a white shirt0.3460790.0a person sitting at a table to eata man sitting at a table\\n0.5989280.5a person sitting at a table to eata man sitting at a table with a woman and a pl...0.5989280.5a person sitting at a table to eata man sitting at a table with a woman and a ch...0.5989280.5a person sitting at a table to eata man sitting at a table with a plate of food ...0.5989280.5
137461a person sitting at a table to eata woman sitting at a table0.5989280.5a person sitting at a table to eata group of people sitting at a table\\n0.5989280.5a person sitting at a table to eata group of people sitting at a table with food...0.5989280.5a person sitting at a table to eata group of people sitting at a table\\n0.5989280.5a person sitting at a table to eata group of people sitting at a table with food...0.5989280.5
180683Two people sitting down to eat foodtwo people sitting at a table0.5989280.5Two people sitting down to eat fooda person sitting at a table\\n0.5989280.5Two people sitting down to eat fooda person sitting at a table with a plate of fo...0.5989280.5Two people sitting down to eat fooda person is sitting at a table\\n0.5989280.5Two people sitting down to eat fooda blurry image of a person sitting at a table ...0.5989280.5
205409man wearing suit and bow tiea man wearing a bow tie0.5986700.5man wearing suit and bow tieman wearing a gray jacket\\n0.5986700.5man wearing suit and bow tiea group of people standing around a man with a...0.3273700.0man wearing suit and bow tiea group of people standing together\\n0.3273700.0man wearing suit and bow tiea group of a man in a suit and bow tie standin...0.7572760.5
21273A truck pulling a white trailer.a truck is behind the train0.0000000.0A truck pulling a white trailer.a truck driving down the road with a trailer a...0.5986140.0A truck pulling a white trailer.a blurry picture of a truck driving down a str...0.5986140.0A truck pulling a white trailer.a truck is driving down a road with a trailer\\n0.5986140.0A truck pulling a white trailer.a truck is driving down the road with a traile...0.5986140.0
21264Truck pulling a trailer behind and to the left...a truck is behind the train0.0000000.0Truck pulling a trailer behind and to the left...a truck driving down the road with a trailer a...0.5986140.0Truck pulling a trailer behind and to the left...a blurry picture of a truck driving down a str...0.5986140.0Truck pulling a trailer behind and to the left...a truck is driving down a road with a trailer\\n0.5986140.0Truck pulling a trailer behind and to the left...a truck is driving down the road with a traile...0.5986140.0
94950A car pulling out.a car on the street0.0000000.0A car pulling out.a car is driving down the street at night\\n0.5986140.0A car pulling out.a blurry image of a street with cars parked on...0.3320130.0A car pulling out.a car is parked on the street\\n0.3320130.0A car pulling out.a blurry image of a street with cars parked on...0.3320130.0
136389two horses pulling the carriagetwo horses pulling a carriage1.0000001.0two horses pulling the carriagea man driving a horse and carriage\\n0.5986140.0two horses pulling the carriagea couple of horses pulling a carriage with a m...1.0000001.0two horses pulling the carriagea horse pulling a carriage\\n1.0000001.0two horses pulling the carriagea man riding a horse drawn carriage with a wom...0.5698810.0
136406two horses pulling wagontwo horses pulling a carriage1.0000001.0two horses pulling wagona man driving a horse drawn carriage\\n0.5986140.0two horses pulling wagona couple of horses pulling a carriage with a m...1.0000001.0two horses pulling wagona man riding a horse\\n0.5698810.0two horses pulling wagona man riding a horse drawn carriage with a wom...0.5698810.0
14635snow covered benches make for a cold seata bench in the snow0.0000000.0snow covered benches make for a cold seata bench covered in snow\\n0.5984070.5snow covered benches make for a cold seata bench covered in snow on a snowy day\\n0.5984070.5snow covered benches make for a cold seata bench covered in snow\\n0.5984070.5snow covered benches make for a cold seata bench covered in snow with a person sitting ...0.6072070.5
9215animal running in grashead of a sheep0.0000000.0animal running in grasa cat is laying on the floor in front of a mir...0.5981880.0animal running in grasa blurry picture of a cat sitting on a bench\\n0.6114940.0animal running in grasa cat is laying on the floor with its head down\\n0.5981880.0animal running in grasa blurry picture of a white dog sitting on a c...0.6114940.0
228895Baby lying on a black changing padthe baby is laying on a black pillow0.5980030.0Baby lying on a black changing pada baby laying on a couch\\n0.5980030.0Baby lying on a black changing pada baby laying on a black bag with a teddy bear\\n0.5980030.0Baby lying on a black changing pada baby is laying on a black suitcase\\n0.5980030.0Baby lying on a black changing pada baby laying on a black cushion with a teddy ...0.5980030.0
192830a baby elephant hiding by an adulta baby elephant0.0000000.0a baby elephant hiding by an adulta large elephant standing in a dirt field\\n0.5979750.0a baby elephant hiding by an adulta blurry picture of an elephant standing next ...0.5979750.0a baby elephant hiding by an adulta large elephant standing in a field\\n0.5979750.0a baby elephant hiding by an adulta close up of an elephant standing in a field ...0.5979750.0
29036Bird is hiding it's head.bird standing in the water0.5979750.0Bird is hiding it's head.a duck standing in the water\\n0.5979750.0Bird is hiding it's head.a blurry picture of a duck standing on a rock\\n0.5979750.0Bird is hiding it's head.a duck is standing in the water\\n0.5979750.0Bird is hiding it's head.a duck with its head down looking at something...0.5426230.0
26194a sheep hiding behind two otherssheep is next to sheep0.0000000.0a sheep hiding behind two otherstwo sheep standing on a hill\\n0.5979750.0a sheep hiding behind two otherstwo sheep standing in a field near a lake\\n0.5979750.0a sheep hiding behind two otherstwo sheep standing on a hill\\n0.5979750.0a sheep hiding behind two otherstwo sheep standing in a field next to a body o...0.5979750.0
85692a man ordering lunchman standing next to truck0.5979070.0a man ordering luncha man standing next to a food truck\\n0.5979070.0a man ordering luncha man standing next to a red truck with a sign...0.5979070.0a man ordering luncha man is standing in front of a food truck\\n0.5979070.0a man ordering luncha man standing in front of a food truck with a...0.5979070.0
173379People behind a plane yelling at itpeople on the grass0.0000000.0People behind a plane yelling at ita group of people standing in front of a plane\\n0.5978880.0People behind a plane yelling at ita blurry image of a couple of people standing ...0.5978880.0People behind a plane yelling at ita group of people standing in front of a plane\\n0.5978880.0People behind a plane yelling at ita group of people standing in a field next to ...0.5978880.0
79582vending booth with vendorman standing in front of building0.5978690.0vending booth with vendora man and woman standing in front of a wooden ...0.5978690.0vending booth with vendora man is standing in front of a small wooden b...0.5978690.0vending booth with vendora man and woman standing in front of a christm...0.5978690.0vending booth with vendora man and woman standing in front of a wooden ...0.5978690.0
79587vending booth with Christmas decorationsa tree in front of a building0.0000000.0vending booth with Christmas decorationsa man standing in front of a wooden stand\\n0.5978690.0vending booth with Christmas decorationsa man is standing in front of a small wooden b...0.5978690.0vending booth with Christmas decorationsa man is walking past a christmas market stall\\n0.4219810.0vending booth with Christmas decorationsa man in a black jacket standing in front of a...0.5978690.0
147455it is raining in the photothe picture is black and white0.0000000.0it is raining in the photoa group of people standing under umbrellas in ...0.5978690.0it is raining in the photoa group of people with umbrellas walking down ...0.4789000.0it is raining in the photoa group of people walking under umbrellas in t...0.4789000.0it is raining in the photoa group of people walking under umbrellas in a...0.4789000.0
237606it is raining outsidepeople standing in a field0.5978690.0it is raining outsidepeople standing in a field\\n0.5978690.0it is raining outsidea group of people standing in a field with umb...0.5978690.0it is raining outsidea group of people standing in a field with umb...0.5978690.0it is raining outsidea group of people standing in a field with umb...0.5978690.0
16019Red, brick building abutting parking lot.a brick building in the background0.0000000.0Red, brick building abutting parking lot.a man is riding a skateboard\\n0.5976080.0Red, brick building abutting parking lot.a man is walking down the street with a red ha...0.6396040.0Red, brick building abutting parking lot.a man is riding a skateboard down a street\\n0.5976080.0Red, brick building abutting parking lot.a man in a red shirt standing next to a brick ...0.7452950.0
78876A very small wave hitting the shorethe waves are white0.0000000.0A very small wave hitting the shorea person on a surfboard riding a wave\\n0.5973660.0A very small wave hitting the shorea close up of a bird flying over a field of gr...0.5750070.0A very small wave hitting the shorea person riding a surfboard on the beach\\n0.5973660.0A very small wave hitting the shorea blurry picture of a person surfing on a wave\\n0.2996010.0
169051white spray hitting shorethe water is white in color0.0000000.0white spray hitting shorea blurry image of a surfer riding a wave\\n0.5973660.0white spray hitting shorea blurry picture of a bird sitting on a tree\\n0.6419270.0white spray hitting shorea man riding a surfboard in the ocean\\n0.5973660.0white spray hitting shorea blurry photo of a surfer riding a wave on a ...0.5973660.0
145362water hitting the shorethe water is white0.0000000.0water hitting the shorea man is riding a skateboard down a street\\n0.5973660.0water hitting the shorea close up of a bird sitting on a tree branch\\n0.6419270.0water hitting the shorea man riding a skateboard on a street\\n0.5973660.0water hitting the shorea man standing in front of a mirror with a tow...0.6282020.0
116306Woman is sitting downa woman talking on a cell phone0.5972390.0Woman is sitting downa woman talking on a cell phone\\n0.5972390.0Woman is sitting downa woman sitting at a table talking on a cell p...1.0000001.0Woman is sitting downa woman sitting on a bench talking on a cell p...1.0000001.0Woman is sitting downa woman sitting in a chair talking on a cell p...1.0000001.0
109084three men talkingthree men sitting on a bench0.5972390.0three men talkingthree men sitting on a bench\\n0.5972390.0three men talkingthree older men sitting on a bench together wi...0.5972390.0three men talkingthree men sitting on a bench\\n0.5972390.0three men talkingthree men sitting on a bench with one man with...0.5972390.0
132127the woman is talking on a phonewhite cell phone in woman's hand0.0000000.0the woman is talking on a phonea woman is sitting on a couch with her legs cr...0.5972390.0the woman is talking on a phonea close up of a person walking down a street\\n0.6272150.0the woman is talking on a phonea woman is holding a cell phone in her hand\\n0.4679300.0the woman is talking on a phonea person wrapped in bandages with a white towe...0.2813860.0
214439A man talking on a mobile phonea man wearing a white t-shirt0.5154100.0A man talking on a mobile phonea man sitting on a chair\\n0.5972390.0A man talking on a mobile phonea man sitting on a chair with a suitcase next ...0.5972390.0A man talking on a mobile phonea man sitting on a chair with luggage next to ...0.5972390.0A man talking on a mobile phonea man sitting on a chair with a suitcase next ...0.5972390.0
136372people talking in front of storetwo people talking on the sidewalk1.0000001.0people talking in front of storea man and woman sitting on a bench\\n0.5972390.0people talking in front of storea man and woman sitting on a bench next to a f...0.5972390.0people talking in front of storea man and woman sitting on a bench\\n0.5972390.0people talking in front of storea man and woman sitting on a bench with a cell...0.5972390.0
72783guy talking on his cellphonea man sitting on the floor0.5972390.0guy talking on his cellphonea man sitting on the floor\\n0.5972390.0guy talking on his cellphonea man sitting on the floor talking on a cell p...1.0000001.0guy talking on his cellphonea man sitting on the ground\\n0.5972390.0guy talking on his cellphonea man sitting on the ground talking on a cell ...1.0000001.0
72788young man in black t-shirt and jeans talking o...a man wearing a black shirt0.5154100.0young man in black t-shirt and jeans talking o...a man sitting on the floor\\n0.5972390.0young man in black t-shirt and jeans talking o...a man sitting on the floor talking on a cell p...1.0000001.0young man in black t-shirt and jeans talking o...a man sitting on the ground\\n0.5972390.0young man in black t-shirt and jeans talking o...a man sitting on the ground talking on a cell ...1.0000001.0
\n", "
" ], "text/plain": [ " sca-ref \\\n", "12766 truck hauling an airplane's cargo \n", "238035 sliding glass door \n", "13302 Tree trunk providing shade \n", "123060 railing is green \n", "96716 person siting on the side \n", "123993 man eating chocolate glazed doughnut with spri... \n", "108952 Catcher crouching down in the dirt \n", "188897 ladels and whisk hanging from ceiling \n", "139590 a cutting boarding leaning against a wall \n", "184970 Guy wearing white shirt and tie \n", "114386 The brown and white bow tie the girl is wearing. \n", "49809 the man is wearing a vest and tie \n", "89413 Man is wearing a suit and tie \n", "169682 A man is wearing a vest and tie \n", "231217 man wearing suit and tie \n", "159392 person wearing an orange and red plaid shirt a... \n", "137457 a person sitting at a table to eat \n", "137455 a person sitting at a table to eat \n", "137462 a person sitting at a table to eat \n", "137461 a person sitting at a table to eat \n", "180683 Two people sitting down to eat food \n", "205409 man wearing suit and bow tie \n", "21273 A truck pulling a white trailer. \n", "21264 Truck pulling a trailer behind and to the left... \n", "94950 A car pulling out. \n", "136389 two horses pulling the carriage \n", "136406 two horses pulling wagon \n", "14635 snow covered benches make for a cold seat \n", "9215 animal running in gras \n", "228895 Baby lying on a black changing pad \n", "192830 a baby elephant hiding by an adult \n", "29036 Bird is hiding it's head. \n", "26194 a sheep hiding behind two others \n", "85692 a man ordering lunch \n", "173379 People behind a plane yelling at it \n", "79582 vending booth with vendor \n", "79587 vending booth with Christmas decorations \n", "147455 it is raining in the photo \n", "237606 it is raining outside \n", "16019 Red, brick building abutting parking lot. \n", "78876 A very small wave hitting the shore \n", "169051 white spray hitting shore \n", "145362 water hitting the shore \n", "116306 Woman is sitting down \n", "109084 three men talking \n", "132127 the woman is talking on a phone \n", "214439 A man talking on a mobile phone \n", "136372 people talking in front of store \n", "72783 guy talking on his cellphone \n", "72788 young man in black t-shirt and jeans talking o... \n", "\n", " sca-pred sca-verb_distance \\\n", "12766 A truck is next to the plane. 0.000000 \n", "238035 the window is open 0.000000 \n", "13302 the trunk of a tree 0.000000 \n", "123060 a green fence on the side of the platform 0.000000 \n", "96716 a man standing in front of a wall 0.742729 \n", "123993 a man eating a donut 0.599321 \n", "108952 the catcher is squatting 0.713759 \n", "188897 utensils hanging from the ceiling 0.599189 \n", "139590 a wooden cutting board 0.000000 \n", "184970 a man wearing a white shirt 0.599068 \n", "114386 the bow tie is black and white 0.000000 \n", "49809 a man wearing a tie 0.599068 \n", "89413 a man wearing a suit 0.599068 \n", "169682 a man wearing a vest 0.599068 \n", "231217 a man wearing a suit 0.599068 \n", "159392 a man wearing a red and white striped shirt 0.599068 \n", "137457 a balding man in a plaid shirt 0.216295 \n", "137455 a woman wearing a gray shirt 0.346079 \n", "137462 a man wearing a white shirt 0.346079 \n", "137461 a woman sitting at a table 0.598928 \n", "180683 two people sitting at a table 0.598928 \n", "205409 a man wearing a bow tie 0.598670 \n", "21273 a truck is behind the train 0.000000 \n", "21264 a truck is behind the train 0.000000 \n", "94950 a car on the street 0.000000 \n", "136389 two horses pulling a carriage 1.000000 \n", "136406 two horses pulling a carriage 1.000000 \n", "14635 a bench in the snow 0.000000 \n", "9215 head of a sheep 0.000000 \n", "228895 the baby is laying on a black pillow 0.598003 \n", "192830 a baby elephant 0.000000 \n", "29036 bird standing in the water 0.597975 \n", "26194 sheep is next to sheep 0.000000 \n", "85692 man standing next to truck 0.597907 \n", "173379 people on the grass 0.000000 \n", "79582 man standing in front of building 0.597869 \n", "79587 a tree in front of a building 0.000000 \n", "147455 the picture is black and white 0.000000 \n", "237606 people standing in a field 0.597869 \n", "16019 a brick building in the background 0.000000 \n", "78876 the waves are white 0.000000 \n", "169051 the water is white in color 0.000000 \n", "145362 the water is white 0.000000 \n", "116306 a woman talking on a cell phone 0.597239 \n", "109084 three men sitting on a bench 0.597239 \n", "132127 white cell phone in woman's hand 0.000000 \n", "214439 a man wearing a white t-shirt 0.515410 \n", "136372 two people talking on the sidewalk 1.000000 \n", "72783 a man sitting on the floor 0.597239 \n", "72788 a man wearing a black shirt 0.515410 \n", "\n", " sca-verb_recall blip2-opt-2.7b-ref \\\n", "12766 0.0 truck hauling an airplane's cargo \n", "238035 0.0 sliding glass door \n", "13302 0.0 Tree trunk providing shade \n", "123060 0.0 railing is green \n", "96716 0.0 person siting on the side \n", "123993 0.5 man eating chocolate glazed doughnut with spri... \n", "108952 0.0 Catcher crouching down in the dirt \n", "188897 0.5 ladels and whisk hanging from ceiling \n", "139590 0.0 a cutting boarding leaning against a wall \n", "184970 0.5 Guy wearing white shirt and tie \n", "114386 0.0 The brown and white bow tie the girl is wearing. \n", "49809 0.5 the man is wearing a vest and tie \n", "89413 0.5 Man is wearing a suit and tie \n", "169682 0.5 A man is wearing a vest and tie \n", "231217 0.5 man wearing suit and tie \n", "159392 0.5 person wearing an orange and red plaid shirt a... \n", "137457 0.0 a person sitting at a table to eat \n", "137455 0.0 a person sitting at a table to eat \n", "137462 0.0 a person sitting at a table to eat \n", "137461 0.5 a person sitting at a table to eat \n", "180683 0.5 Two people sitting down to eat food \n", "205409 0.5 man wearing suit and bow tie \n", "21273 0.0 A truck pulling a white trailer. \n", "21264 0.0 Truck pulling a trailer behind and to the left... \n", "94950 0.0 A car pulling out. \n", "136389 1.0 two horses pulling the carriage \n", "136406 1.0 two horses pulling wagon \n", "14635 0.0 snow covered benches make for a cold seat \n", "9215 0.0 animal running in gras \n", "228895 0.0 Baby lying on a black changing pad \n", "192830 0.0 a baby elephant hiding by an adult \n", "29036 0.0 Bird is hiding it's head. \n", "26194 0.0 a sheep hiding behind two others \n", "85692 0.0 a man ordering lunch \n", "173379 0.0 People behind a plane yelling at it \n", "79582 0.0 vending booth with vendor \n", "79587 0.0 vending booth with Christmas decorations \n", "147455 0.0 it is raining in the photo \n", "237606 0.0 it is raining outside \n", "16019 0.0 Red, brick building abutting parking lot. \n", "78876 0.0 A very small wave hitting the shore \n", "169051 0.0 white spray hitting shore \n", "145362 0.0 water hitting the shore \n", "116306 0.0 Woman is sitting down \n", "109084 0.0 three men talking \n", "132127 0.0 the woman is talking on a phone \n", "214439 0.0 A man talking on a mobile phone \n", "136372 1.0 people talking in front of store \n", "72783 0.0 guy talking on his cellphone \n", "72788 0.0 young man in black t-shirt and jeans talking o... \n", "\n", " blip2-opt-2.7b-pred \\\n", "12766 a truck is driving down the road\\n \n", "238035 a man is holding a white towel\\n \n", "13302 a man is standing in front of a building\\n \n", "123060 a train is pulling into a station\\n \n", "96716 a man is painting a wall\\n \n", "123993 a man eating a doughnut\\n \n", "108952 a baseball player is throwing a ball\\n \n", "188897 a kitchen with a mixer and a whisk hanging fro... \n", "139590 a wooden cutting board sitting on top of a sink\\n \n", "184970 a man wearing a white shirt\\n \n", "114386 a man wearing a bow tie\\n \n", "49809 man wearing a black vest\\n \n", "89413 man wearing a suit\\n \n", "169682 a man wearing a vest\\n \n", "231217 a man wearing a tie\\n \n", "159392 a man wearing a tie\\n \n", "137457 a group of people sitting at a table\\n \n", "137455 a group of people sitting at a table\\n \n", "137462 a man sitting at a table\\n \n", "137461 a group of people sitting at a table\\n \n", "180683 a person sitting at a table\\n \n", "205409 man wearing a gray jacket\\n \n", "21273 a truck driving down the road with a trailer a... \n", "21264 a truck driving down the road with a trailer a... \n", "94950 a car is driving down the street at night\\n \n", "136389 a man driving a horse and carriage\\n \n", "136406 a man driving a horse drawn carriage\\n \n", "14635 a bench covered in snow\\n \n", "9215 a cat is laying on the floor in front of a mir... \n", "228895 a baby laying on a couch\\n \n", "192830 a large elephant standing in a dirt field\\n \n", "29036 a duck standing in the water\\n \n", "26194 two sheep standing on a hill\\n \n", "85692 a man standing next to a food truck\\n \n", "173379 a group of people standing in front of a plane\\n \n", "79582 a man and woman standing in front of a wooden ... \n", "79587 a man standing in front of a wooden stand\\n \n", "147455 a group of people standing under umbrellas in ... \n", "237606 people standing in a field\\n \n", "16019 a man is riding a skateboard\\n \n", "78876 a person on a surfboard riding a wave\\n \n", "169051 a blurry image of a surfer riding a wave\\n \n", "145362 a man is riding a skateboard down a street\\n \n", "116306 a woman talking on a cell phone\\n \n", "109084 three men sitting on a bench\\n \n", "132127 a woman is sitting on a couch with her legs cr... \n", "214439 a man sitting on a chair\\n \n", "136372 a man and woman sitting on a bench\\n \n", "72783 a man sitting on the floor\\n \n", "72788 a man sitting on the floor\\n \n", "\n", " blip2-opt-2.7b-verb_distance blip2-opt-2.7b-verb_recall \\\n", "12766 0.599886 0.0 \n", "238035 0.599672 0.0 \n", "13302 0.599651 0.0 \n", "123060 0.599623 0.0 \n", "96716 0.599327 0.0 \n", "123993 0.599321 0.5 \n", "108952 0.599297 0.0 \n", "188897 0.599189 0.5 \n", "139590 0.599170 0.0 \n", "184970 0.599068 0.5 \n", "114386 0.599068 0.5 \n", "49809 0.599068 0.5 \n", "89413 0.599068 0.5 \n", "169682 0.599068 0.5 \n", "231217 0.599068 0.5 \n", "159392 0.599068 0.5 \n", "137457 0.598928 0.5 \n", "137455 0.598928 0.5 \n", "137462 0.598928 0.5 \n", "137461 0.598928 0.5 \n", "180683 0.598928 0.5 \n", "205409 0.598670 0.5 \n", "21273 0.598614 0.0 \n", "21264 0.598614 0.0 \n", "94950 0.598614 0.0 \n", "136389 0.598614 0.0 \n", "136406 0.598614 0.0 \n", "14635 0.598407 0.5 \n", "9215 0.598188 0.0 \n", "228895 0.598003 0.0 \n", "192830 0.597975 0.0 \n", "29036 0.597975 0.0 \n", "26194 0.597975 0.0 \n", "85692 0.597907 0.0 \n", "173379 0.597888 0.0 \n", "79582 0.597869 0.0 \n", "79587 0.597869 0.0 \n", "147455 0.597869 0.0 \n", "237606 0.597869 0.0 \n", "16019 0.597608 0.0 \n", "78876 0.597366 0.0 \n", "169051 0.597366 0.0 \n", "145362 0.597366 0.0 \n", "116306 0.597239 0.0 \n", "109084 0.597239 0.0 \n", "132127 0.597239 0.0 \n", "214439 0.597239 0.0 \n", "136372 0.597239 0.0 \n", "72783 0.597239 0.0 \n", "72788 0.597239 0.0 \n", "\n", " blip2-opt-2.7b-coco-ref \\\n", "12766 truck hauling an airplane's cargo \n", "238035 sliding glass door \n", "13302 Tree trunk providing shade \n", "123060 railing is green \n", "96716 person siting on the side \n", "123993 man eating chocolate glazed doughnut with spri... \n", "108952 Catcher crouching down in the dirt \n", "188897 ladels and whisk hanging from ceiling \n", "139590 a cutting boarding leaning against a wall \n", "184970 Guy wearing white shirt and tie \n", "114386 The brown and white bow tie the girl is wearing. \n", "49809 the man is wearing a vest and tie \n", "89413 Man is wearing a suit and tie \n", "169682 A man is wearing a vest and tie \n", "231217 man wearing suit and tie \n", "159392 person wearing an orange and red plaid shirt a... \n", "137457 a person sitting at a table to eat \n", "137455 a person sitting at a table to eat \n", "137462 a person sitting at a table to eat \n", "137461 a person sitting at a table to eat \n", "180683 Two people sitting down to eat food \n", "205409 man wearing suit and bow tie \n", "21273 A truck pulling a white trailer. \n", "21264 Truck pulling a trailer behind and to the left... \n", "94950 A car pulling out. \n", "136389 two horses pulling the carriage \n", "136406 two horses pulling wagon \n", "14635 snow covered benches make for a cold seat \n", "9215 animal running in gras \n", "228895 Baby lying on a black changing pad \n", "192830 a baby elephant hiding by an adult \n", "29036 Bird is hiding it's head. \n", "26194 a sheep hiding behind two others \n", "85692 a man ordering lunch \n", "173379 People behind a plane yelling at it \n", "79582 vending booth with vendor \n", "79587 vending booth with Christmas decorations \n", "147455 it is raining in the photo \n", "237606 it is raining outside \n", "16019 Red, brick building abutting parking lot. \n", "78876 A very small wave hitting the shore \n", "169051 white spray hitting shore \n", "145362 water hitting the shore \n", "116306 Woman is sitting down \n", "109084 three men talking \n", "132127 the woman is talking on a phone \n", "214439 A man talking on a mobile phone \n", "136372 people talking in front of store \n", "72783 guy talking on his cellphone \n", "72788 young man in black t-shirt and jeans talking o... \n", "\n", " blip2-opt-2.7b-coco-pred \\\n", "12766 a truck is carrying boxes on the back of it\\n \n", "238035 a blurry picture of a cell phone with a red li... \n", "13302 a blurry image of a giraffe standing in a field\\n \n", "123060 a man is walking down a sidewalk with a skateb... \n", "96716 a woman is sitting on a chair in a bathroom\\n \n", "123993 a man eating a doughnut with sprinkles on it\\n \n", "108952 a blurry picture of a table with a bunch of st... \n", "188897 a close up of a mixer with a whisk and a bowl\\n \n", "139590 a man is sitting on a chair with a book in fro... \n", "184970 a man in a white shirt and tie standing next t... \n", "114386 a close up of a man wearing a white shirt and ... \n", "49809 a man in a suit and tie taking a selfie\\n \n", "89413 a man in a suit and tie taking a selfie\\n \n", "169682 a man in a sweater vest and tie sitting at a t... \n", "231217 a man in a suit and tie sitting at a table wit... \n", "159392 a young man in a tie and plaid shirt holding a... \n", "137457 a group of people sitting at a table with a pl... \n", "137455 a group of people sitting at a table with plat... \n", "137462 a man sitting at a table with a woman and a pl... \n", "137461 a group of people sitting at a table with food... \n", "180683 a person sitting at a table with a plate of fo... \n", "205409 a group of people standing around a man with a... \n", "21273 a blurry picture of a truck driving down a str... \n", "21264 a blurry picture of a truck driving down a str... \n", "94950 a blurry image of a street with cars parked on... \n", "136389 a couple of horses pulling a carriage with a m... \n", "136406 a couple of horses pulling a carriage with a m... \n", "14635 a bench covered in snow on a snowy day\\n \n", "9215 a blurry picture of a cat sitting on a bench\\n \n", "228895 a baby laying on a black bag with a teddy bear\\n \n", "192830 a blurry picture of an elephant standing next ... \n", "29036 a blurry picture of a duck standing on a rock\\n \n", "26194 two sheep standing in a field near a lake\\n \n", "85692 a man standing next to a red truck with a sign... \n", "173379 a blurry image of a couple of people standing ... \n", "79582 a man is standing in front of a small wooden b... \n", "79587 a man is standing in front of a small wooden b... \n", "147455 a group of people with umbrellas walking down ... \n", "237606 a group of people standing in a field with umb... \n", "16019 a man is walking down the street with a red ha... \n", "78876 a close up of a bird flying over a field of gr... \n", "169051 a blurry picture of a bird sitting on a tree\\n \n", "145362 a close up of a bird sitting on a tree branch\\n \n", "116306 a woman sitting at a table talking on a cell p... \n", "109084 three older men sitting on a bench together wi... \n", "132127 a close up of a person walking down a street\\n \n", "214439 a man sitting on a chair with a suitcase next ... \n", "136372 a man and woman sitting on a bench next to a f... \n", "72783 a man sitting on the floor talking on a cell p... \n", "72788 a man sitting on the floor talking on a cell p... \n", "\n", " blip2-opt-2.7b-coco-verb_distance blip2-opt-2.7b-coco-verb_recall \\\n", "12766 0.726537 0.0 \n", "238035 0.000000 0.0 \n", "13302 0.599651 0.0 \n", "123060 0.495665 0.0 \n", "96716 0.656225 0.0 \n", "123993 0.599321 0.5 \n", "108952 0.000000 0.0 \n", "188897 0.000000 0.0 \n", "139590 0.599170 0.0 \n", "184970 0.757276 0.5 \n", "114386 0.599068 0.5 \n", "49809 0.734603 0.5 \n", "89413 0.734603 0.5 \n", "169682 0.776905 0.5 \n", "231217 0.776905 0.5 \n", "159392 0.352133 0.0 \n", "137457 0.598928 0.5 \n", "137455 0.598928 0.5 \n", "137462 0.598928 0.5 \n", "137461 0.598928 0.5 \n", "180683 0.598928 0.5 \n", "205409 0.327370 0.0 \n", "21273 0.598614 0.0 \n", "21264 0.598614 0.0 \n", "94950 0.332013 0.0 \n", "136389 1.000000 1.0 \n", "136406 1.000000 1.0 \n", "14635 0.598407 0.5 \n", "9215 0.611494 0.0 \n", "228895 0.598003 0.0 \n", "192830 0.597975 0.0 \n", "29036 0.597975 0.0 \n", "26194 0.597975 0.0 \n", "85692 0.597907 0.0 \n", "173379 0.597888 0.0 \n", "79582 0.597869 0.0 \n", "79587 0.597869 0.0 \n", "147455 0.478900 0.0 \n", "237606 0.597869 0.0 \n", "16019 0.639604 0.0 \n", "78876 0.575007 0.0 \n", "169051 0.641927 0.0 \n", "145362 0.641927 0.0 \n", "116306 1.000000 1.0 \n", "109084 0.597239 0.0 \n", "132127 0.627215 0.0 \n", "214439 0.597239 0.0 \n", "136372 0.597239 0.0 \n", "72783 1.000000 1.0 \n", "72788 1.000000 1.0 \n", "\n", " blip2-opt-6.7b-ref \\\n", "12766 truck hauling an airplane's cargo \n", "238035 sliding glass door \n", "13302 Tree trunk providing shade \n", "123060 railing is green \n", "96716 person siting on the side \n", "123993 man eating chocolate glazed doughnut with spri... \n", "108952 Catcher crouching down in the dirt \n", "188897 ladels and whisk hanging from ceiling \n", "139590 a cutting boarding leaning against a wall \n", "184970 Guy wearing white shirt and tie \n", "114386 The brown and white bow tie the girl is wearing. \n", "49809 the man is wearing a vest and tie \n", "89413 Man is wearing a suit and tie \n", "169682 A man is wearing a vest and tie \n", "231217 man wearing suit and tie \n", "159392 person wearing an orange and red plaid shirt a... \n", "137457 a person sitting at a table to eat \n", "137455 a person sitting at a table to eat \n", "137462 a person sitting at a table to eat \n", "137461 a person sitting at a table to eat \n", "180683 Two people sitting down to eat food \n", "205409 man wearing suit and bow tie \n", "21273 A truck pulling a white trailer. \n", "21264 Truck pulling a trailer behind and to the left... \n", "94950 A car pulling out. \n", "136389 two horses pulling the carriage \n", "136406 two horses pulling wagon \n", "14635 snow covered benches make for a cold seat \n", "9215 animal running in gras \n", "228895 Baby lying on a black changing pad \n", "192830 a baby elephant hiding by an adult \n", "29036 Bird is hiding it's head. \n", "26194 a sheep hiding behind two others \n", "85692 a man ordering lunch \n", "173379 People behind a plane yelling at it \n", "79582 vending booth with vendor \n", "79587 vending booth with Christmas decorations \n", "147455 it is raining in the photo \n", "237606 it is raining outside \n", "16019 Red, brick building abutting parking lot. \n", "78876 A very small wave hitting the shore \n", "169051 white spray hitting shore \n", "145362 water hitting the shore \n", "116306 Woman is sitting down \n", "109084 three men talking \n", "132127 the woman is talking on a phone \n", "214439 A man talking on a mobile phone \n", "136372 people talking in front of store \n", "72783 guy talking on his cellphone \n", "72788 young man in black t-shirt and jeans talking o... \n", "\n", " blip2-opt-6.7b-pred \\\n", "12766 a truck with a load of boxes on the back\\n \n", "238035 a person is holding a cell phone in front of a... \n", "13302 a person is standing in front of a building\\n \n", "123060 a train is coming\\n \n", "96716 a man standing in front of a large white board\\n \n", "123993 a man eating a donut\\n \n", "108952 a baseball player is swinging a bat\\n \n", "188897 a kitchen with a stove and a mixer\\n \n", "139590 a sink with a faucet and a wooden cutting board\\n \n", "184970 a man wearing a white shirt\\n \n", "114386 a man wearing a bow tie\\n \n", "49809 a man wearing a vest\\n \n", "89413 man wearing a suit\\n \n", "169682 a man wearing a vest\\n \n", "231217 a man wearing a suit and tie\\n \n", "159392 a man wearing a tie\\n \n", "137457 a group of people sitting at a table\\n \n", "137455 a woman wearing a green sweater\\n \n", "137462 a man sitting at a table with a woman and a ch... \n", "137461 a group of people sitting at a table\\n \n", "180683 a person is sitting at a table\\n \n", "205409 a group of people standing together\\n \n", "21273 a truck is driving down a road with a trailer\\n \n", "21264 a truck is driving down a road with a trailer\\n \n", "94950 a car is parked on the street\\n \n", "136389 a horse pulling a carriage\\n \n", "136406 a man riding a horse\\n \n", "14635 a bench covered in snow\\n \n", "9215 a cat is laying on the floor with its head down\\n \n", "228895 a baby is laying on a black suitcase\\n \n", "192830 a large elephant standing in a field\\n \n", "29036 a duck is standing in the water\\n \n", "26194 two sheep standing on a hill\\n \n", "85692 a man is standing in front of a food truck\\n \n", "173379 a group of people standing in front of a plane\\n \n", "79582 a man and woman standing in front of a christm... \n", "79587 a man is walking past a christmas market stall\\n \n", "147455 a group of people walking under umbrellas in t... \n", "237606 a group of people standing in a field with umb... \n", "16019 a man is riding a skateboard down a street\\n \n", "78876 a person riding a surfboard on the beach\\n \n", "169051 a man riding a surfboard in the ocean\\n \n", "145362 a man riding a skateboard on a street\\n \n", "116306 a woman sitting on a bench talking on a cell p... \n", "109084 three men sitting on a bench\\n \n", "132127 a woman is holding a cell phone in her hand\\n \n", "214439 a man sitting on a chair with luggage next to ... \n", "136372 a man and woman sitting on a bench\\n \n", "72783 a man sitting on the ground\\n \n", "72788 a man sitting on the ground\\n \n", "\n", " blip2-opt-6.7b-verb_distance blip2-opt-6.7b-verb_recall \\\n", "12766 0.000000 0.0 \n", "238035 0.599672 0.0 \n", "13302 0.599651 0.0 \n", "123060 0.418080 0.0 \n", "96716 0.742729 0.0 \n", "123993 0.599321 0.5 \n", "108952 0.648913 0.0 \n", "188897 0.000000 0.0 \n", "139590 0.000000 0.0 \n", "184970 0.599068 0.5 \n", "114386 0.599068 0.5 \n", "49809 0.599068 0.5 \n", "89413 0.599068 0.5 \n", "169682 0.599068 0.5 \n", "231217 1.000000 1.0 \n", "159392 0.599068 0.5 \n", "137457 0.598928 0.5 \n", "137455 0.346079 0.0 \n", "137462 0.598928 0.5 \n", "137461 0.598928 0.5 \n", "180683 0.598928 0.5 \n", "205409 0.327370 0.0 \n", "21273 0.598614 0.0 \n", "21264 0.598614 0.0 \n", "94950 0.332013 0.0 \n", "136389 1.000000 1.0 \n", "136406 0.569881 0.0 \n", "14635 0.598407 0.5 \n", "9215 0.598188 0.0 \n", "228895 0.598003 0.0 \n", "192830 0.597975 0.0 \n", "29036 0.597975 0.0 \n", "26194 0.597975 0.0 \n", "85692 0.597907 0.0 \n", "173379 0.597888 0.0 \n", "79582 0.597869 0.0 \n", "79587 0.421981 0.0 \n", "147455 0.478900 0.0 \n", "237606 0.597869 0.0 \n", "16019 0.597608 0.0 \n", "78876 0.597366 0.0 \n", "169051 0.597366 0.0 \n", "145362 0.597366 0.0 \n", "116306 1.000000 1.0 \n", "109084 0.597239 0.0 \n", "132127 0.467930 0.0 \n", "214439 0.597239 0.0 \n", "136372 0.597239 0.0 \n", "72783 0.597239 0.0 \n", "72788 0.597239 0.0 \n", "\n", " blip2-opt-6.7b-coco-ref \\\n", "12766 truck hauling an airplane's cargo \n", "238035 sliding glass door \n", "13302 Tree trunk providing shade \n", "123060 railing is green \n", "96716 person siting on the side \n", "123993 man eating chocolate glazed doughnut with spri... \n", "108952 Catcher crouching down in the dirt \n", "188897 ladels and whisk hanging from ceiling \n", "139590 a cutting boarding leaning against a wall \n", "184970 Guy wearing white shirt and tie \n", "114386 The brown and white bow tie the girl is wearing. \n", "49809 the man is wearing a vest and tie \n", "89413 Man is wearing a suit and tie \n", "169682 A man is wearing a vest and tie \n", "231217 man wearing suit and tie \n", "159392 person wearing an orange and red plaid shirt a... \n", "137457 a person sitting at a table to eat \n", "137455 a person sitting at a table to eat \n", "137462 a person sitting at a table to eat \n", "137461 a person sitting at a table to eat \n", "180683 Two people sitting down to eat food \n", "205409 man wearing suit and bow tie \n", "21273 A truck pulling a white trailer. \n", "21264 Truck pulling a trailer behind and to the left... \n", "94950 A car pulling out. \n", "136389 two horses pulling the carriage \n", "136406 two horses pulling wagon \n", "14635 snow covered benches make for a cold seat \n", "9215 animal running in gras \n", "228895 Baby lying on a black changing pad \n", "192830 a baby elephant hiding by an adult \n", "29036 Bird is hiding it's head. \n", "26194 a sheep hiding behind two others \n", "85692 a man ordering lunch \n", "173379 People behind a plane yelling at it \n", "79582 vending booth with vendor \n", "79587 vending booth with Christmas decorations \n", "147455 it is raining in the photo \n", "237606 it is raining outside \n", "16019 Red, brick building abutting parking lot. \n", "78876 A very small wave hitting the shore \n", "169051 white spray hitting shore \n", "145362 water hitting the shore \n", "116306 Woman is sitting down \n", "109084 three men talking \n", "132127 the woman is talking on a phone \n", "214439 A man talking on a mobile phone \n", "136372 people talking in front of store \n", "72783 guy talking on his cellphone \n", "72788 young man in black t-shirt and jeans talking o... \n", "\n", " blip2-opt-6.7b-coco-pred \\\n", "12766 a truck with a load of furniture on the back o... \n", "238035 a blurry picture of a man standing in front of... \n", "13302 a man standing in the shade of a tree with a s... \n", "123060 a man walking down a sidewalk next to a green ... \n", "96716 a woman standing in front of a large white boa... \n", "123993 a man with glasses eating a doughnut with spri... \n", "108952 a blurry image of a baseball player in a unifo... \n", "188897 a ceiling fan and a metal whisk hanging from a... \n", "139590 a close up of a sink with a faucet and a towel\\n \n", "184970 a man in a white shirt and tie standing outside\\n \n", "114386 a person wearing a brown bow tie with a white ... \n", "49809 a man in a vest and tie taking a selfie\\n \n", "89413 a man in a suit taking a selfie in a mirror\\n \n", "169682 a man in a sweater and tie sitting at a table\\n \n", "231217 a man in a suit and tie sitting at a table\\n \n", "159392 a man in a tie and plaid shirt standing next t... \n", "137457 a group of people sitting at a table with food... \n", "137455 a group of people sitting at a table eating fo... \n", "137462 a man sitting at a table with a plate of food ... \n", "137461 a group of people sitting at a table with food... \n", "180683 a blurry image of a person sitting at a table ... \n", "205409 a group of a man in a suit and bow tie standin... \n", "21273 a truck is driving down the road with a traile... \n", "21264 a truck is driving down the road with a traile... \n", "94950 a blurry image of a street with cars parked on... \n", "136389 a man riding a horse drawn carriage with a wom... \n", "136406 a man riding a horse drawn carriage with a wom... \n", "14635 a bench covered in snow with a person sitting ... \n", "9215 a blurry picture of a white dog sitting on a c... \n", "228895 a baby laying on a black cushion with a teddy ... \n", "192830 a close up of an elephant standing in a field ... \n", "29036 a duck with its head down looking at something... \n", "26194 two sheep standing in a field next to a body o... \n", "85692 a man standing in front of a food truck with a... \n", "173379 a group of people standing in a field next to ... \n", "79582 a man and woman standing in front of a wooden ... \n", "79587 a man in a black jacket standing in front of a... \n", "147455 a group of people walking under umbrellas in a... \n", "237606 a group of people standing in a field with umb... \n", "16019 a man in a red shirt standing next to a brick ... \n", "78876 a blurry picture of a person surfing on a wave\\n \n", "169051 a blurry photo of a surfer riding a wave on a ... \n", "145362 a man standing in front of a mirror with a tow... \n", "116306 a woman sitting in a chair talking on a cell p... \n", "109084 three men sitting on a bench with one man with... \n", "132127 a person wrapped in bandages with a white towe... \n", "214439 a man sitting on a chair with a suitcase next ... \n", "136372 a man and woman sitting on a bench with a cell... \n", "72783 a man sitting on the ground talking on a cell ... \n", "72788 a man sitting on the ground talking on a cell ... \n", "\n", " blip2-opt-6.7b-coco-verb_distance blip2-opt-6.7b-coco-verb_recall \n", "12766 0.000000 0.0 \n", "238035 0.612457 0.0 \n", "13302 0.599651 0.0 \n", "123060 0.495665 0.0 \n", "96716 0.742729 0.0 \n", "123993 0.599321 0.5 \n", "108952 0.563707 0.0 \n", "188897 0.599189 0.5 \n", "139590 0.000000 0.0 \n", "184970 0.757276 0.5 \n", "114386 0.599068 0.5 \n", "49809 0.734603 0.5 \n", "89413 0.272746 0.0 \n", "169682 0.776905 0.5 \n", "231217 0.776905 0.5 \n", "159392 0.288216 0.0 \n", "137457 0.598928 0.5 \n", "137455 0.739933 0.5 \n", "137462 0.598928 0.5 \n", "137461 0.598928 0.5 \n", "180683 0.598928 0.5 \n", "205409 0.757276 0.5 \n", "21273 0.598614 0.0 \n", "21264 0.598614 0.0 \n", "94950 0.332013 0.0 \n", "136389 0.569881 0.0 \n", "136406 0.569881 0.0 \n", "14635 0.607207 0.5 \n", "9215 0.611494 0.0 \n", "228895 0.598003 0.0 \n", "192830 0.597975 0.0 \n", "29036 0.542623 0.0 \n", "26194 0.597975 0.0 \n", "85692 0.597907 0.0 \n", "173379 0.597888 0.0 \n", "79582 0.597869 0.0 \n", "79587 0.597869 0.0 \n", "147455 0.478900 0.0 \n", "237606 0.597869 0.0 \n", "16019 0.745295 0.0 \n", "78876 0.299601 0.0 \n", "169051 0.597366 0.0 \n", "145362 0.628202 0.0 \n", "116306 1.000000 1.0 \n", "109084 0.597239 0.0 \n", "132127 0.281386 0.0 \n", "214439 0.597239 0.0 \n", "136372 0.597239 0.0 \n", "72783 1.000000 1.0 \n", "72788 1.000000 1.0 " ] }, "execution_count": 72, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.loc[(df['blip2-opt-2.7b-verb_distance'] <= 0.6) & (df['blip2-opt-2.7b-verb_distance'] >= 0.4)].sort_values(by='blip2-opt-2.7b-verb_distance', ascending=False).head(50)" ] } ], "metadata": { "kernelspec": { "display_name": "sca-v2", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.18" } }, "nbformat": 4, "nbformat_minor": 2 }