Upload 2 files

.gitattributes

@@ -41,3 +41,4 @@ general_blip_train_llava_70ORI_30COCO_swift.json filter=lfs diff=lfs merge=lfs -
 general_blip_train_llava_70ORI_30COCO_swift_A100.json filter=lfs diff=lfs merge=lfs -text
 general_blip_train_llava_coco_swift_A100.json filter=lfs diff=lfs merge=lfs -text
 general_blip_train_llava_swift_A100.json filter=lfs diff=lfs merge=lfs -text
+general_blip_train_llava_imgh_swift_multi_A100.json filter=lfs diff=lfs merge=lfs -text

aitw_to_swift.ipynb

@@ -0,0 +1,382 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os \n",
+    "import json\n",
+    "\n",
+    "def read_json(file_path): \n",
+    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
+    "        data = json.load(file)\n",
+    "    return data\n",
+    "\n",
+    "def write_json(file_path, data):\n",
+    "    with open(file_path, 'w', encoding='utf-8') as file:\n",
+    "        json.dump(data, file, ensure_ascii=False, indent=4)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### 更换路径"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# data = read_json('/code/Data/m4_instruct_annotations.json')\n",
+    "# data = read_json('/code/Data/general_blip_train_llava_imgh.json')\n",
+    "data = read_json('/code/Data/general_blip_train_llava_swift.json')\n",
+    "# data = read_json('/code/Data/general_blip_test_llava_swift.json')\n",
+    "\n",
+    "# data = read_json('/code/LLaVA/data/json/general_blip_train_llava.json')\n",
+    "# data = read_json('/code/LLaVA/data/json/all_blip_train_llava_coco.json')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'conversations': [{'from': 'human',\n",
+      "                    'value': '<image>\\n'\n",
+      "                             'Previous Actions: Goal: Open a new Chrome '\n",
+      "                             'private window'},\n",
+      "                   {'from': 'gpt',\n",
+      "                    'value': 'Action Plan: '\n",
+      "                             '[DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\\n'\n",
+      "                             '; Action Decision: \"action_type\": \"DUAL_POINT\", '\n",
+      "                             '\"touch_point\": \"[0.7761, 0.7089]\", \"lift_point\": '\n",
+      "                             '\"[0.7761, 0.7089]\", \"typed_text\": \"\"'}],\n",
+      " 'id': 'general_blip_0',\n",
+      " 'image': 'blip/general_texts_splits/10_1.png'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pprint\n",
+    "\n",
+    "pprint.pprint(data[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'images': ['/code/Auto-GUI/dataset/blip/general_texts_splits/10_1.png'],\n",
+      " 'query': '<<image>\\n'\n",
+      "          'Previous Actions: Goal: Open a new Chrome private window>55555',\n",
+      " 'response': 'Action Plan: '\n",
+      "             '[DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\\n'\n",
+      "             '; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": '\n",
+      "             '\"[0.7761, 0.7089]\", \"lift_point\": \"[0.7761, 0.7089]\", '\n",
+      "             '\"typed_text\": \"\"'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pprint\n",
+    "\n",
+    "pprint.pprint(data[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "8831"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for index, i  in enumerate(data):\n",
+    "\n",
+    "    # ################## A6000 ##################\n",
+    "    # data[index]['images'][0] = '/data/home/zbz5349/WorkSpace/LLaVA/data/blip' + data[index]['images'][0][27:]\n",
+    "\n",
+    "    ################## H100 ##################\n",
+    "    data[index]['images'][0] = '/gpu02home/zbz5349/ICLR_2024/LLaVA_Mobile_V1/data/blip' + data[index]['images'][0][27:]\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# write_json('/code/Data/general_blip_train_llava_swift_a6000.json', data)\n",
+    "# write_json('/code/Data/general_blip_train_llava_swift_H100.json', data)\n",
+    "write_json('/code/Data/general_blip_test_llava_swift_H100.json', data)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "-----"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 重构格式 / Single Image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "path = '/code/LLaVA/data/json/general_blip_train_llava.json'\n",
+    "# path = '/code/LLaVA/data/json/general_blip_train_llava_coco.json'\n",
+    "# path = '/code/LLaVA/data/json/general_blip_train_llava_70ORI_30COCO.json'\n",
+    "\n",
+    "# path = '/code/LLaVA/data/json/all_blip_train_llava.json'\n",
+    "# path = '/code/LLaVA/data/json/all_blip_train_llava_coco.json'\n",
+    "\n",
+    "data = read_json(path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "new_data = []\n",
+    "for index, i in enumerate(data):\n",
+    "\n",
+    "    temp = {}\n",
+    "    temp['query'] = i['conversations'][0]['value']\n",
+    "    temp['response'] = i['conversations'][1]['value']\n",
+    "    temp['images'] = []\n",
+    "\n",
+    "    ############## H100\n",
+    "    # temp_image_path = '/gpu02home/zbz5349/ICLR_2024/LLaVA_Mobile_V1/data/' + i['image'] \n",
+    "\n",
+    "    ############## A100\n",
+    "    temp_image_path = '/data/zbz5349/ICLR_2024/data/' + i['image']\n",
+    "    \n",
+    "    temp['images'].append(temp_image_path)\n",
+    "\n",
+    "    new_data.append(temp)\n",
+    "    # pprint.pprint(temp)\n",
+    "    # break"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "new_path = path.split('.')[0] + '_swift_A100.json'\n",
+    "write_json(new_path, new_data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'/code/LLaVA/data/json/all_blip_train_llava_swift.json'"
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "new_path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 重构格式 / Multi Image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "path = '/code/Data/general_blip_train_llava_imgh.json'\n",
+    " \n",
+    "data = read_json(path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pprint\n",
+    "\n",
+    "\n",
+    "def replace_hashes(text):\n",
+    "    # 使用 rsplit 分割并替换最后五个 ### 为 xxx\n",
+    "    parts = text.rsplit('###', 4)\n",
+    "    # 将替换后的文本重新拼接\n",
+    "    return '<image>'.join(parts)\n",
+    "\n",
+    "def ensure_five_xxx(text):\n",
+    "    # 统计文本中 'xxx' 的出现次数\n",
+    "    count = text.count('<image>')\n",
+    "    if count < 5:\n",
+    "        # 计算还需要补充多少个 'xxx'\n",
+    "        missing_xxx = 5 - count\n",
+    "        # 在文本末尾添加足够的 'xxx'\n",
+    "        text += ' <image>' * missing_xxx\n",
+    "    \n",
+    "    return text\n",
+    "\n",
+    "\n",
+    "new_data = []\n",
+    "for index, i in enumerate(data):\n",
+    "\n",
+    "    temp = {}\n",
+    "    temp['query'] = i['conversations'][0]['value']\n",
+    "    temp['response'] = i['conversations'][1]['value']\n",
+    "    \n",
+    "    ############## A100 ##############\n",
+    "    temp_image_path_list = []\n",
+    "    for w in i['image_history']:\n",
+    "        temp_image_path = '/data/zbz5349/ICLR_2024/data/' + w\n",
+    "        temp_image_path_list.append(temp_image_path)\n",
+    "\n",
+    "    temp['images'] = temp_image_path_list \n",
+    "    \n",
+    "    new_temp = temp['query'].split('\"action_type\"')\n",
+    "    new_temp = ' ###\\n  \"action_type\"'.join(new_temp)\n",
+    "    # pprint.pprint(new_temp) \n",
+    "    new_temp = replace_hashes(new_temp)\n",
+    "    new_temp = ensure_five_xxx(new_temp)\n",
+    "    # pprint.pprint(new_temp) \n",
+    "    # break\n",
+    "\n",
+    "    temp['query'] = new_temp\n",
+    "    new_data.append(temp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'images': ['/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13_3.png',\n",
+      "            '/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13_2.png',\n",
+      "            '/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13_1.png',\n",
+      "            '/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13_1.png',\n",
+      "            '/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13_1.png'],\n",
+      " 'query': '<image>\\n'\n",
+      "          'Previous Actions:  <image>\\n'\n",
+      "          '  \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", '\n",
+      "          '\"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"  <image>\\n'\n",
+      "          '  \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7649, 0.6773]\", '\n",
+      "          '\"lift_point\": \"[0.7649, 0.6773]\", \"typed_text\": \"\" Goal: Open a new '\n",
+      "          'Chrome window <image> <image>',\n",
+      " 'response': 'Action Plan: [STATUS_TASK_COMPLETE]\\n'\n",
+      "             '; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", '\n",
+      "             '\"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", '\n",
+      "             '\"typed_text\": \"\"'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "# data[0]\n",
+    "\n",
+    "pprint.pprint(new_data[11]) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "new_path = path.split('.')[0] + '_swift_multi_A100.json'\n",
+    "write_json(new_path, new_data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "llava",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

general_blip_train_llava_imgh_swift_multi_A100.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbe3be1b9f4b1e9c2774835fab06ea53f16b1e90415d3c0ca019260646e89ffd
+size 95345344