Upload folder using huggingface_hub

.env

@@ -0,0 +1 @@
+PALM="AIzaSyBVojf3nBKO_UITOwZtDVyAejW_2Qne1KY"

README.md

@@ -1,12 +1,6 @@
 ---
-title: Sonitycom
-emoji: 👁
-colorFrom: yellow
-colorTo: purple
+title: sonitycom
+app_file: demo.py
 sdk: gradio
 sdk_version: 4.7.1
-app_file: app.py
-pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

demo.py

@@ -0,0 +1,166 @@
+import numpy as np
+import pandas as pd
+import time
+from sentence_transformers import SentenceTransformer
+from redis.commands.search.field import VectorField
+from redis.commands.search.field import TextField
+from redis.commands.search.field import TagField
+from redis.commands.search.query import Query
+import redis
+from tqdm import tqdm
+import google.generativeai as palm
+import pandas as pd
+from langchain.chains import LLMChain
+
+
+from langchain.prompts import PromptTemplate
+
+import os
+
+import gradio as gr
+import io
+
+from langchain.llms import GooglePalm
+import pandas as pd
+#from yolopandas import pd
+
+from langchain.embeddings import GooglePalmEmbeddings
+from langchain.memory import ConversationBufferMemory
+from dotenv import load_dotenv
+
+load_dotenv()
+
+redis_conn = redis.Redis(
+  host='redis-15860.c322.us-east-1-2.ec2.cloud.redislabs.com',
+  port=15860,
+  password='PVnvSZI5nISPsrxxhCHZF3pfZWI7YAIG')
+
+'''
+df = pd.read_csv("coms3.csv")
+
+
+print(list(df))
+
+print(df['item_keywords'].sample(2))
+
+company_metadata = df.to_dict(orient='index')
+
+
+model = SentenceTransformer('sentence-transformers/all-distilroberta-v1')
+
+
+item_keywords =  [company_metadata[i]['item_keywords']  for i in company_metadata.keys()]
+item_keywords_vectors = []
+for sentence in tqdm(item_keywords):
+    s = model.encode(sentence)
+    item_keywords_vectors.append(s)
+
+print(company_metadata[0])
+
+def load_vectors(client, company_metadata, vector_dict, vector_field_name):
+    p = client.pipeline(transaction=False)
+    for index in company_metadata.keys():
+        #hash key
+        #print(index)
+        #print(company_metadata[index]['company_l_id'])
+        try:
+            key=str('company:'+ str(index)+ ':' + company_metadata[index]['primary_key'])
+        except:
+            print(key)
+            continue
+
+
+        #hash values
+        item_metadata = company_metadata[index]
+        item_keywords_vector = vector_dict[index].astype(np.float32).tobytes()
+        item_metadata[vector_field_name]=item_keywords_vector
+
+        # HSET
+        p.hset(key,mapping=item_metadata)
+
+    p.execute()
+
+def create_flat_index (redis_conn,vector_field_name,number_of_vectors, vector_dimensions=512, distance_metric='L2'):
+    redis_conn.ft().create_index([
+        VectorField(vector_field_name, "FLAT", {"TYPE": "FLOAT32", "DIM": vector_dimensions, "DISTANCE_METRIC": distance_metric, "INITIAL_CAP": number_of_vectors, "BLOCK_SIZE":number_of_vectors }),
+        TagField("company_l_id"),
+        TextField("company_name"),
+        TextField("item_keywords"),
+        TagField("industry")
+    ])
+
+ITEM_KEYWORD_EMBEDDING_FIELD='item_keyword_vector'
+TEXT_EMBEDDING_DIMENSION=768
+NUMBER_COMPANIES=1000
+
+print ('Loading and Indexing + ' +  str(NUMBER_COMPANIES) + 'companies')
+
+#flush all data
+redis_conn.flushall()
+
+#create flat index & load vectors
+create_flat_index(redis_conn, ITEM_KEYWORD_EMBEDDING_FIELD,NUMBER_COMPANIES,TEXT_EMBEDDING_DIMENSION,'COSINE')
+load_vectors(redis_conn,company_metadata,item_keywords_vectors,ITEM_KEYWORD_EMBEDDING_FIELD)
+'''
+model = SentenceTransformer('sentence-transformers/all-distilroberta-v1')
+ITEM_KEYWORD_EMBEDDING_FIELD='item_keyword_vector'
+TEXT_EMBEDDING_DIMENSION=768
+NUMBER_PRODUCTS=1000
+
+prompt = PromptTemplate(
+    input_variables=["company_description"],
+    template='Create comma seperated company keywords to perform a query on a company dataset for this user input'
+)
+
+template = """You are a chatbot. Be kind, detailed and nice. Present the given queried search result in a nice way as answer to the user input. dont ask questions back! just take the given context
+
+{chat_history}
+Human: {user_question}
+Chatbot:
+"""
+
+prompt = PromptTemplate(
+    input_variables=["chat_history", "user_question"],
+    template=template
+)
+chat_history= ""
+def answer(user_question):
+    llm = GooglePalm(temperature=0, google_api_key=os.environ['PALM'])
+    chain = LLMChain(llm=llm, prompt=prompt)
+    keywords = chain.run({'user_question':user_question, 'chat_history':chat_history})
+
+    topK=3
+#vectorize the query
+    query_vector = model.encode(keywords).astype(np.float32).tobytes()
+
+    q = Query(f'*=>[KNN {topK} @{ITEM_KEYWORD_EMBEDDING_FIELD} $vec_param AS vector_score]').sort_by('vector_score').paging(0,topK).return_fields('vector_score','item_name','item_id','item_keywords').dialect(2)
+    params_dict = {"vec_param": query_vector}
+
+    #Execute the query
+    results = redis_conn.ft().search(q, query_params = params_dict)
+
+    full_result_string = ''
+    for company in results.docs:
+        full_result_string += company.company_name + ' ' + company.item_keywords + ' ' + company.company_l_id + "\n\n\n"
+
+    memory = ConversationBufferMemory(memory_key="chat_history")
+    llm_chain = LLMChain(
+    llm=llm,
+    prompt=prompt,
+    verbose=False,
+    memory=memory,
+)
+
+
+    ans = llm_chain.predict(user_msg= f"{full_result_string} ---\n\n {user_question}")
+
+    return ans
+
+demo = gr.Interface(
+
+    fn=answer,
+    inputs=["text"],
+    outputs=["text"],
+    title="Ask Sonity",
+)
+demo.launch(share=True)

requirements.txt

@@ -0,0 +1,114 @@
+aiofiles==23.2.1
+aiohttp==3.8.6
+aiosignal==1.3.1
+annotated-types==0.6.0
+anyio==3.7.1
+appdirs==1.4.4
+async-timeout==4.0.3
+attrs==23.1.0
+beautifulsoup4==4.12.2
+Brotli==1.1.0
+cachetools==5.3.2
+certifi==2023.7.22
+charset-normalizer==3.3.2
+click==8.1.7
+dataclasses-json==0.6.2
+duckduckgo-search==3.9.4
+exceptiongroup==1.1.3
+filelock==3.13.1
+frozendict==2.3.8
+frozenlist==1.4.0
+fsspec==2023.10.0
+google-ai-generativelanguage==0.1.0
+google-api-core==2.14.0
+google-auth==2.23.4
+google-generativeai==0.1.0rc1
+googleapis-common-protos==1.61.0
+greenlet==3.0.1
+grpcio==1.59.3
+grpcio-status==1.59.3
+h11==0.14.0
+h2==4.1.0
+hpack==4.0.0
+html5lib==1.1
+httpcore==1.0.2
+httpx==0.25.1
+huggingface-hub==0.19.3
+hyperframe==6.0.1
+idna==3.4
+install==1.3.5
+Jinja2==3.1.2
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+langchain==0.0.333
+langsmith==0.0.63
+lxml==4.9.3
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+mpmath==1.3.0
+multidict==6.0.4
+multitasking==0.0.11
+mypy-extensions==1.0.0
+networkx==3.1
+nltk==3.8.1
+numpy==1.24.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.18.1
+nvidia-nvjitlink-cu12==12.3.52
+nvidia-nvtx-cu12==12.1.105
+packaging==23.2
+pandas==2.0.3
+peewee==3.17.0
+Pillow==10.1.0
+proto-plus==1.22.3
+protobuf==4.25.1
+psycopg2-binary==2.9.9
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pydantic==2.4.2
+pydantic-core==2.10.1
+python-dateutil==2.8.2
+pytz==2023.3.post1
+PyYAML==6.0.1
+redis==5.0.1
+regex==2023.10.3
+requests==2.31.0
+rsa==4.9
+safetensors==0.4.0
+scikit-learn==1.3.2
+scipy==1.10.1
+sentence-transformers==2.2.2
+sentencepiece==0.1.99
+simplejson==3.19.2
+six==1.16.0
+sniffio==1.3.0
+socksio==1.0.0
+soupsieve==2.5
+SQLAlchemy==2.0.23
+sympy==1.12
+tenacity==8.2.3
+threadpoolctl==3.2.0
+tokenizers==0.15.0
+torch==2.1.1
+torchvision==0.16.1
+tqdm==4.66.1
+transformers==4.35.2
+triton==2.1.0
+typing-extensions==4.8.0
+typing-inspect==0.9.0
+tzdata==2023.3
+urllib3==2.0.7
+webencodings==0.5.1
+wikipedia==1.4.0
+yahoo-finance==1.4.0
+yarl==1.9.2
+yfinance==0.2.31