first commit

main.py

@@ -1,11 +1,19 @@
 from fastapi import FastAPI
+from service import VectorStore
+
+PROJECT_NAME = "filecoin"
 
 app = FastAPI()
+vector_store = VectorStore(PROJECT_NAME)
 
 @app.get("/")
 def read_root():
     return {"Hello": "World!"}
 
-@app.get("/")
-def read_root():
-    return {"Hello": "World!"}
+@app.get("/vector/search")
+def vector_search(inputs):
+    return vector_store.search(inputs)
+
+@app.get("/vector/marginal_search")
+def vector_marginal_search(inputs):
+    return vector_store.marginal_search(inputs)

requirements.txt

@@ -1,7 +1,91 @@
-fastapi==0.74.*
-requests==2.27.*
-sentencepiece==0.1.*
-torch==1.11.*
-transformers==4.*
-uvicorn[standard]==0.17.*
-
+aiofiles==23.2.1
+aiohttp==3.9.1
+aiosignal==1.3.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==3.7.1
+attrs==23.1.0
+beautifulsoup4==4.12.2
+bs4==0.0.1
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+contourpy==1.2.0
+cycler==0.12.1
+dataclasses-json==0.6.3
+distro==1.8.0
+dnspython==2.4.2
+fastapi==0.105.0
+ffmpy==0.3.1
+filelock==3.13.1
+fonttools==4.47.0
+frozenlist==1.4.1
+fsspec==2023.12.2
+gradio==4.11.0
+gradio_client==0.7.3
+greenlet==3.0.2
+h11==0.14.0
+httpcore==1.0.2
+httpx==0.25.2
+huggingface-hub==0.19.4
+idna==3.6
+importlib-resources==6.1.1
+Jinja2==3.1.2
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.2
+kiwisolver==1.4.5
+langchain==0.0.332
+langsmith==0.0.72
+loguru==0.7.2
+lxml==4.9.4
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib==3.8.2
+mdurl==0.1.2
+multidict==6.0.4
+mypy-extensions==1.0.0
+numpy==1.26.2
+openai==1.6.0
+orjson==3.9.10
+packaging==23.2
+pandas==2.1.4
+Pillow==10.1.0
+pinecone-client==2.2.4
+pydantic==2.5.2
+pydantic_core==2.14.5
+pydub==0.25.1
+Pygments==2.17.2
+pyparsing==3.1.1
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3.post1
+PyYAML==6.0.1
+referencing==0.32.0
+regex==2023.10.3
+requests==2.31.0
+rich==13.7.0
+rpds-py==0.15.2
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.5
+SQLAlchemy==2.0.23
+starlette==0.27.0
+tenacity==8.2.3
+tiktoken==0.5.2
+tomlkit==0.12.0
+toolz==0.12.0
+tqdm==4.66.1
+typer==0.9.0
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+tzdata==2023.3
+urllib3==2.1.0
+uvicorn==0.24.0.post1
+websockets==11.0.3
+yarl==1.9.4

service/vector_store.py

@@ -7,6 +7,7 @@ from langchain.docstore.document import Document
 from langchain.schema.embeddings import Embeddings
 from langchain.vectorstores.utils import DistanceStrategy, maximal_marginal_relevance
 import numpy as np
+import json
 import logging
 
 logger = logging.getLogger(__name__)
@@ -27,8 +28,32 @@ class VectorStore(Pinecone):
     def add_docs(self,docs):
         if self.index_name not in pinecone.list_indexes():
             pinecone.create_index(name=self.index_name, metric="cosine", dimension=self.dims)
+        for doc in docs:
+            metadata = doc.metadata
+            for k in metadata:
+                if metadata.get(k) is None:
+                    metadata[k] = 'unknown'
         Pinecone.from_documents(docs, self.embeddings, index_name=self.index_name)
 
+    def search(self,query,ret_json=True):
+        docs = self.similarity_search(query)
+        if ret_json:
+            return self.jsonfy(docs)
+        else:
+            return docs
+
+    def marginal_search(self,query,ret_json=True):
+        docs = self.max_marginal_relevance_search(query)
+        if ret_json:
+            return self.jsonfy(docs)
+        else:
+            return docs
+        
+    def jsonfy(self,docs):
+        docs = [doc.dict() for doc in docs]
+        docs = json.dumps(docs)
+        return docs
+
     def similarity_search_by_vector_with_score(
         self,
         embedding: List[float],

test/vector_store.py

@@ -2,6 +2,6 @@ from service import VectorStore
 
 index_name = 'filecoin'
 vector_store = VectorStore(index_name)
-print(vector_store.similarity_search("what is filecoin"))
+print(vector_store.search("what is filecoin",ret_json=True))
 
-print(vector_store.max_marginal_relevance_search("what is filecoin"))
+print(vector_store.marginal_search("what is filecoin",ret_json=True))