Estudando o DB redis

Index_Creation/1_index_creation_and_query.py

@@ -0,0 +1,119 @@
+"""
+Data Scientist.: Dr.Eddy Giusepe Chirinos Isidro
+
+
+Similaridade do Vetor
+=====================
+Usamos os vetores (Embeddings) em sistemas de recomendação, pesquisa de 
+imagens e vídeos, recuperação de documentos e Q&A 🤗.
+
+Versão: 1.0.0
+Data: 29/07/2023
+Autor: Dr.Eddy Giusepe
+
+Método de execução:
+                   $ python 1_index_creation.py
+"""
+import redis
+from redis.commands.search.field import TagField, VectorField
+from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.query import Query
+
+r = redis.Redis(host="localhost", port=6379)
+
+INDEX_NAME = "index"                              # Vector Index Name
+DOC_PREFIX = "doc:"                               # RediSearch Key Prefix for the Index
+
+def create_index(vector_dimensions: int):
+    try:
+        # check to see if index exists
+        r.ft(INDEX_NAME).info()
+        print("Index already exists!")
+    except:
+        # schema
+        schema = (
+            TagField("tag"),                       # Tag Field Name
+            TagField("id"),
+            VectorField("vector",                  # Vector Field Name
+                "FLAT", {                          # Vector Index Type: FLAT or HNSW
+                    "TYPE": "FLOAT32",             # FLOAT32 or FLOAT64
+                    "DIM": vector_dimensions,      # Number of Vector Dimensions
+                    "DISTANCE_METRIC": "COSINE",   # Vector Search Distance Metric
+                }
+            ),
+        )
+
+        # index Definition
+        definition = IndexDefinition(prefix=[DOC_PREFIX], index_type=IndexType.HASH)
+
+        # create Index
+        r.ft(INDEX_NAME).create_index(fields=schema, definition=definition)
+
+# define vector dimensions
+VECTOR_DIMENSIONS = 1536
+
+texts = [
+     "Hoje é realmente um ótimo dia!",
+     "O cachorro da porta ao lado late muito alto.",
+     "Meu gato escapou e saiu antes que eu pudesse fechar a porta.",
+     "Amanhã deve chover e trovejar."
+]
+
+# delete index
+#r.ft(INDEX_NAME).dropindex(delete_documents=True)
+
+# make a new one
+create_index(vector_dimensions=VECTOR_DIMENSIONS)
+
+
+import openai
+import os
+import numpy as np
+from dotenv import load_dotenv, find_dotenv
+_ = load_dotenv(find_dotenv()) # read local .env file
+
+openai.api_key  = os.environ['OPENAI_API_KEY']
+
+
+# Create Embeddings with OpenAI text-embedding-ada-002
+# https://openai.com/blog/new-and-improved-embedding-model
+response = openai.Embedding.create(input=texts, engine="text-embedding-ada-002")
+#print(response)
+embeddings = np.array([r["embedding"] for r in response["data"]], dtype=np.float32)
+
+# Write to Redis
+pipe = r.pipeline()
+for i, embedding in enumerate(embeddings):
+    pipe.hset(f"doc:{i}", mapping = {
+        "vector": embedding.tobytes(),
+        "content": texts[i],
+        "tag": "Eddy"
+    })
+res = pipe.execute()
+#print("🤗")
+#print(embeddings)
+
+#text = "animals"
+text = "Eu gosto muito de pets, especificamente de gatos"
+
+# create query embedding
+response = openai.Embedding.create(input=[text], engine="text-embedding-ada-002")
+#print(response["data"][0])
+query_embedding = np.array([r["embedding"] for r in response["data"]], dtype=np.float32)[0]
+#print("🤗🤗")
+#print(len(query_embedding))
+
+filter_query = "*"
+# query for similar documents
+query = (
+    Query(f"{filter_query}=>[KNN 3 @vector $vec as score]")
+     .sort_by("score")
+     .return_fields("content", "tag", "score")
+     .paging(0, 3)
+     .dialect(2)
+)
+
+query_params = {"vec": query_embedding.tobytes()}
+response = r.ft(INDEX_NAME).search(query, query_params).docs
+print("🤗🤗🤗")
+print(response)

Index_Creation/query_by_index.py

@@ -0,0 +1,92 @@
+"""
+Data Scientist.: Dr.Eddy Giusepe Chirinos Isidro
+
+
+Similaridade do Vetor
+=====================
+Usamos os vetores (Embeddings) em sistemas de recomendação, pesquisa de 
+imagens e vídeos, recuperação de documentos e Q&A 🤗.
+
+Versão: 1.0.0
+Data: 29/07/2023
+Autor: Dr.Eddy Giusepe
+
+Método de execução:
+                   $ python query_by_index.py
+"""
+import redis
+from redis.commands.search.query import Query # Para fazer a pesquisa dos K vizinhos mais próximos
+import numpy as np
+import openai
+import os
+from dotenv import load_dotenv, find_dotenv
+_ = load_dotenv(find_dotenv()) # read local .env file
+openai.api_key  = os.environ['OPENAI_API_KEY']
+
+# Conecte-se ao servidor Redis local
+redis_host = "localhost"
+redis_port = 6379
+r = redis.Redis(host=redis_host, port=redis_port)
+
+# Texto da pergunta
+text = "Qual é o horário de atendimento no codhab?"
+
+# Crie o embedding da pergunta
+response = openai.Embedding.create(input=text, engine="text-embedding-ada-002")
+embedding_text = response['data'][0]['embedding']
+#query_embedding = np.array([r["embedding"] for r in response["data"]], dtype=np.float32)[0]
+array_embedding = np.array(embedding_text, dtype=np.float32)
+query_embedding = array_embedding.tobytes() # Tem que estar em Bytes ... o REDEIS pede assim.
+
+
+
+# Defina a lista de índices a serem consultados em ordem de prioridade
+index_list = ["secretaria_de_saude_do_df", "detran", "codhab"]
+
+# Crie a query considerando a busca por similaridade nos índices
+filter_query = " | ".join(f"(@vector [{index}] $vec){' ' + index if i > 0 else ''}" for i, index in enumerate(index_list))
+query = (
+    Query(f"{filter_query} =>[KNN 3 @vector $vec as score]")
+    .sort_by("@score")
+    .paging(0, 3)
+    .return_fields("score")
+    .dialect(2)
+)
+
+# Adicione o comando de busca KNN com o query_embedding à (@tag:{ EddyGiusepe })pipeline
+pipe = r.pipeline()
+pipe.execute_command("FT.SEARCH", "codhab", query, "RETURN", 1, "@vector", "VECTORS", query_embedding)  # Substitua "myIndex" pelo nome do seu índice
+result = pipe.execute()
+
+print(result)
+
+#teste = r.hgetall("doc:codhab:86ce630aa248409191e01b33a5f9fedf")
+teste = r.hgetall("doc:codhab:86ce630aa248409191e01b33a5f9fedf")
+print(teste.keys())
+print(teste[b'content_vector']) # Este é o Embedding, não precisa de "utf-8"
+print(teste[b'metadata'].decode('utf-8'))
+print(teste[b'content'].decode('utf-8'))
+
+import json
+dic = json.loads(teste[b'metadata'].decode('utf-8'))
+
+print(dic["Pergunta"])
+print(dic["Resposta"])
+print(dic["Fonte"])
+
+
+keys = r.keys("doc:codhab:*")
+print(len(keys))
+# if len(result["data"]) == 0:
+#     # Se não houver resultados nos índices, faça uma busca em todos eles usando "*"
+#     query = (
+#         Query(f"* =>[KNN 3 @vector $vec as score]")
+#         .sort_by("@score")
+#         .paging(0, 3)
+#         .return_fields("content", "tag", "score")
+#         .dialect(2)
+#     )
+#     result = query.execute(set={"vec": query_embedding.tolist()})
+
+# # Resultados
+# print(result["data"])