Upload 4 files

1_build_text_image_search_engine9.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Build a Milvus Powered Text-Image Search Engine in Minutes
+# 
+# This notebook illustrates how to build an text-image search engine from scratch using [Milvus](https://milvus.io/). Milvus is the most advanced open-source vector database built for AI applications and supports nearest neighbor embedding search across tens of millions of entries. We'll go through text-image search procedures and evaluate the performance. Moreover, we managed to make the core functionality as simple as a dozen lines of code, with which you can start hacking your own image search engine.
+
+# ## Preparation
+# ### Install Dependencies
+# First we need to install dependencies such as pymilvus, towhee, gradio and opencv-python.
+
+# In[7]:
+
+
+#! python -m pip install -q pymilvus towhee gradio opencv-python
+# pip3 install transformers #pip3 install torch #pip3 install torchvision
+
+
+# ### Prepare the data
+# 
+# The dataset used in this demo is a subset of the ImageNet dataset (100 classes, 10 images for each class), and the dataset is available via [Github](https://github.com/towhee-io/examples/releases/download/data/reverse_image_search.zip). 
+# 
+# The dataset is organized as follows:
+# - **train**: directory of candidate images;
+# - **test**: directory of test images;
+# - **reverse_image_search.csv**: a csv file containing an ***id***, ***path***, and ***label*** for each image;
+# 
+# Let's take a quick look:
+
+# In[8]:
+
+
+get_ipython().system(' curl -L https://github.com/towhee-io/examples/releases/download/data/reverse_image_search.zip -O')
+get_ipython().system(' unzip -q -o reverse_image_search.zip')
+
+
+# In[9]:
+
+
+import pandas as pd
+
+df = pd.read_csv('reverse_image_search.csv')
+df.head()
+
+
+# In[10]:
+
+
+dfnew=df.loc[df['label'] == 'hunger_games']
+dfnew
+
+
+# To use the dataset for text-image search, let's first define some helper function:
+# 
+# - **read_images(results)**: read images by image IDs;
+
+# In[11]:
+
+
+import cv2
+from towhee.types.image import Image
+
+id_img = df.set_index('id')['path'].to_dict()
+def read_images(results):
+    imgs = []
+    for re in results:
+        path = id_img[re.id]
+        imgs.append(Image(cv2.imread(path), 'BGR'))
+    return imgs
+
+
+# ### Create a Milvus Collection
+# 
+# Before getting started, please make sure you have [installed milvus](https://milvus.io/docs/v2.0.x/install_standalone-docker.md). Let's first create a `text_image_search` collection that uses the [L2 distance metric](https://milvus.io/docs/v2.0.x/metric.md#Euclidean-distance-L2) and an [IVF_FLAT index](https://milvus.io/docs/v2.0.x/index.md#IVF_FLAT).
+
+# In[12]:
+
+
+from pymilvus import connections, FieldSchema, CollectionSchema, DataType, Collection, utility
+
+def create_milvus_collection(collection_name, dim):
+    connections.connect(host='127.0.0.1', port='19530')
+    
+    if utility.has_collection(collection_name):
+        utility.drop_collection(collection_name)
+    
+    fields = [
+    FieldSchema(name='id', dtype=DataType.INT64, descrition='ids', is_primary=True, auto_id=False),
+    FieldSchema(name='embedding', dtype=DataType.FLOAT_VECTOR, descrition='embedding vectors', dim=dim)
+    ]
+    schema = CollectionSchema(fields=fields, description='text image search')
+    collection = Collection(name=collection_name, schema=schema)
+
+    # create IVF_FLAT index for collection.
+    index_params = {
+        'metric_type':'L2',
+        'index_type':"IVF_FLAT",
+        'params':{"nlist":512}
+    }
+    collection.create_index(field_name="embedding", index_params=index_params)
+    return collection
+
+collection = create_milvus_collection('text_image_search', 512)
+
+
+# ## Text Image Search
+# 
+# In this section, we'll show how to build our text-image search engine using Milvus. The basic idea behind our text-image search is the extract embeddings from images and texts using a deep neural network and compare the embeddings with those stored in Milvus.
+# 
+# We use [Towhee](https://towhee.io/), a machine learning framework that allows for creating data processing pipelines, and it also provides predefined operators which implement insert and query operation in Milvus.
+# 
+# <img src="./workflow.png" width = "60%" height = "60%" align=center />
+
+# ### Generate image and text embeddings with CLIP
+# 
+
+# This operator extracts features for image or text with [CLIP](https://openai.com/blog/clip/) which can generate embeddings for text and image by jointly training an image encoder and text encoder to maximize the cosine similarity.
+
+# In[13]:
+
+
+from towhee import ops, pipe, DataCollection
+import numpy as np
+
+
+# In[14]:
+
+
+###. This section needs to have the teddy.png in the folder. Else it will throw an error.
+p = (
+    pipe.input('path')
+    .map('path', 'img', ops.image_decode.cv2('rgb'))
+    .map('img', 'vec', ops.image_text_embedding.clip(model_name='clip_vit_base_patch16', modality='image'))
+    .map('vec', 'vec', lambda x: x / np.linalg.norm(x))
+    .output('img', 'vec')
+)
+
+DataCollection(p('./teddy.png')).show()
+
+
+# In[15]:
+
+
+p2 = (
+    pipe.input('text')
+    .map('text', 'vec', ops.image_text_embedding.clip(model_name='clip_vit_base_patch16', modality='text'))
+    .map('vec', 'vec', lambda x: x / np.linalg.norm(x))
+    .output('text', 'vec')
+)
+
+DataCollection(p2("A teddybear on a skateboard in Times Square.")).show()
+
+
+# Here is detailed explanation of the code:
+# 
+# - `map('path', 'img', ops.image_decode.cv2_rgb('rgb'))`: for each row from the data, read and decode the image at `path` and put the pixel data into column `img`;
+# 
+# - `map('img', 'vec', ops.image_text_embedding.clip(model_name='clip_vit_base_patch16',modality='image'/'text')`: extract image or text embedding feature with `ops.image_text_embedding.clip`, an operator from the [Towhee hub](https://towhee.io/image-text-embedding/clip) . This operator supports seveal models including `clip_vit_base_patch16`,`clip_vit_base_patch32`,`clip_vit_large_patch14`,`clip_vit_large_patch14_336`,etc.
+
+# ### Load Image Embeddings into Milvus
+# 
+# We first extract embeddings from images with `clip_vit_base_patch16` model and insert the embeddings into Milvus for indexing. Towhee provides a [method-chaining style API](https://towhee.readthedocs.io/en/main/index.html) so that users can assemble a data processing pipeline with operators. 
+
+# In[16]:
+
+
+### If cuda. not installed. replace delete device =0  in (in. this code it is deleted.)
+### .map('img', 'vec', ops.image_text_embedding.clip(model_name='clip_vit_base_patch16', modality='image', device=0))
+### This code takes about 5 to 8 mins to execute. on a CPU
+
+
+# In[17]:
+
+
+get_ipython().run_cell_magic('time', '', "collection = create_milvus_collection('text_image_search', 512)\n\ndef read_csv(csv_path, encoding='utf-8-sig'):\n    import csv\n    with open(csv_path, 'r', encoding=encoding) as f:\n        data = csv.DictReader(f)\n        for line in data:\n            yield int(line['id']), line['path']\n\np3 = (\n    pipe.input('csv_file')\n    .flat_map('csv_file', ('id', 'path'), read_csv)\n    .map('path', 'img', ops.image_decode.cv2('rgb'))\n    .map('img', 'vec', ops.image_text_embedding.clip(model_name='clip_vit_base_patch16', modality='image'))\n    .map('vec', 'vec', lambda x: x / np.linalg.norm(x))\n    .map(('id', 'vec'), (), ops.ann_insert.milvus_client(host='127.0.0.1', port='19530', collection_name='text_image_search'))\n    .output()\n)\n\nret = p3('reverse_image_search.csv')\n")
+
+
+# In[18]:
+
+
+collection.load()
+
+
+# In[19]:
+
+
+print('Total number of inserted data is {}.'.format(collection.num_entities))
+
+
+# ### Query Matched Images from Milvus
+
+# Now that embeddings for candidate images have been inserted into Milvus, we can query across it for nearest neighbors. Again, we use Towhee to load the input Text, compute an embedding vector, and use the vector as a query for Milvus. Because Milvus only outputs image IDs and distance values, we provide a `read_images` function to get the original image based on IDs and display.
+
+# In[20]:
+
+
+import pandas as pd
+import cv2
+
+def read_image(image_ids):
+    df = pd.read_csv('reverse_image_search.csv')
+    id_img = df.set_index('id')['path'].to_dict()
+    imgs = []
+    decode = ops.image_decode.cv2('rgb')
+    for image_id in image_ids:
+        path = id_img[image_id]
+        imgs.append(decode(path))
+    return imgs
+
+
+p4 = (
+    pipe.input('text')
+    .map('text', 'vec', ops.image_text_embedding.clip(model_name='clip_vit_base_patch16', modality='text'))
+    .map('vec', 'vec', lambda x: x / np.linalg.norm(x))
+    .map('vec', 'result', ops.ann_search.milvus_client(host='127.0.0.1', port='19530', collection_name='text_image_search', limit=5))
+    .map('result', 'image_ids', lambda x: [item[0] for item in x])
+    .map('image_ids', 'images', read_image)
+    .output('text', 'images')
+)
+
+DataCollection(p4("A white dog")).show()
+DataCollection(p4("A black dog")).show()
+
+
+# ## Release a Showcase
+
+# We've done an excellent job on the core functionality of our text-image search engine. Now it's time to build a showcase with interface. [Gradio](https://gradio.app/) is a great tool for building demos. With Gradio, we simply need to wrap the data processing pipeline via a `search_in_milvus` function:
+
+# In[21]:
+
+
+search_pipeline = (
+    pipe.input('text')
+    .map('text', 'vec', ops.image_text_embedding.clip(model_name='clip_vit_base_patch16', modality='text'))
+    .map('vec', 'vec', lambda x: x / np.linalg.norm(x))
+    .map('vec', 'result', ops.ann_search.milvus_client(host='127.0.0.1', port='19530', collection_name='text_image_search', limit=5))
+    .map('result', 'image_ids', lambda x: [item[0] for item in x])
+    .output('image_ids')
+)
+
+def search(text):
+    df = pd.read_csv('reverse_image_search.csv')
+    id_img = df.set_index('id')['path'].to_dict()
+    imgs = []
+    image_ids = search_pipeline(text).to_list()[0][0]
+    return [id_img[image_id] for image_id in image_ids]
+
+
+# In[22]:
+
+
+import gradio
+
+interface = gradio.Interface(search, 
+                             gradio.inputs.Textbox(lines=1),
+                             [gradio.outputs.Image(type="filepath", label=None) for _ in range(5)]
+                            )
+
+interface.launch()
+
+
+# In[ ]:
+
+
+
+

docker-compose.yml

@@ -0,0 +1,49 @@
+version: '3.5'
+
+services:
+  etcd:
+    container_name: milvus-etcd
+    image: quay.io/coreos/etcd:v3.5.5
+    environment:
+      - ETCD_AUTO_COMPACTION_MODE=revision
+      - ETCD_AUTO_COMPACTION_RETENTION=1000
+      - ETCD_QUOTA_BACKEND_BYTES=4294967296
+      - ETCD_SNAPSHOT_COUNT=50000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
+    command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+
+  minio:
+    container_name: milvus-minio
+    image: minio/minio:RELEASE.2023-03-20T20-16-18Z
+    environment:
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
+    command: minio server /minio_data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  standalone:
+    container_name: milvus-standalone
+    image: milvusdb/milvus:v2.2.10
+    command: ["milvus", "run", "standalone"]
+    environment:
+      ETCD_ENDPOINTS: etcd:2379
+      MINIO_ADDRESS: minio:9000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
+    ports:
+      - "19530:19530"
+      - "9091:9091"
+    depends_on:
+      - "etcd"
+      - "minio"
+
+networks:
+  default:
+    name: milvus

requirements.txt

@@ -0,0 +1,123 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.0.1
+anyio==3.7.0
+appnope==0.1.3
+asttokens==2.2.1
+async-timeout==4.0.2
+attrs==23.1.0
+backcall==0.2.0
+bleach==6.0.0
+certifi==2023.5.7
+charset-normalizer==3.1.0
+click==8.1.3
+comm==0.1.3
+contourpy==1.1.0
+cycler==0.11.0
+debugpy==1.6.7
+decorator==5.1.1
+docutils==0.20.1
+executing==1.2.0
+fastapi==0.97.0
+ffmpy==0.3.0
+filelock==3.12.2
+fonttools==4.40.0
+frozenlist==1.3.3
+fsspec==2023.6.0
+gradio==3.35.2
+gradio_client==0.2.7
+grpcio==1.53.0
+grpcio-tools==1.53.0
+h11==0.14.0
+httpcore==0.17.2
+httpx==0.24.1
+huggingface-hub==0.15.1
+idna==3.4
+importlib-metadata==6.7.0
+ipykernel==6.23.2
+ipython==8.14.0
+jaraco.classes==3.2.3
+jedi==0.18.2
+Jinja2==3.1.2
+jsonschema==4.17.3
+jupyter_client==8.2.0
+jupyter_core==5.3.1
+keyring==24.0.1
+kiwisolver==1.4.4
+linkify-it-py==2.0.2
+markdown-it-py==2.2.0
+MarkupSafe==2.1.3
+matplotlib==3.7.1
+matplotlib-inline==0.1.6
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mmh3==4.0.0
+more-itertools==9.1.0
+mpmath==1.3.0
+multidict==6.0.4
+nest-asyncio==1.5.6
+networkx==3.1
+numpy==1.25.0
+opencv-python==4.7.0.72
+orjson==3.9.1
+packaging==23.1
+pandas==2.0.2
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==9.5.0
+pkginfo==1.9.6
+platformdirs==3.7.0
+prompt-toolkit==3.0.38
+protobuf==4.23.3
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pydantic==1.10.9
+pydub==0.25.1
+Pygments==2.15.1
+pymilvus==2.2.5
+pyparsing==3.1.0
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+PyYAML==6.0
+pyzmq==25.1.0
+readme-renderer==40.0
+regex==2023.6.3
+requests==2.31.0
+requests-toolbelt==1.0.0
+rfc3986==2.0.0
+rich==13.4.2
+safetensors==0.3.1
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+stack-data==0.6.2
+starlette==0.27.0
+sympy==1.12
+tabulate==0.9.0
+tenacity==8.2.2
+tokenizers==0.13.3
+toolz==0.12.0
+torch==2.0.1
+torchvision==0.15.2
+tornado==6.3.2
+towhee==1.1.0
+tqdm==4.65.0
+traitlets==5.9.0
+transformers==4.30.2
+twine==4.0.2
+typing_extensions==4.6.3
+tzdata==2023.3
+uc-micro-py==1.0.2
+ujson==5.8.0
+urllib3==2.0.3
+uvicorn==0.22.0
+wcwidth==0.2.6
+webencodings==0.5.1
+websockets==11.0.3
+yarl==1.9.2
+zipp==3.15.0