build_knowledge.py

# Load .jsonl
import json
from langchain_chroma import Chroma
from langchain_huggingface import HuggingFaceEmbeddings
from langchain.tools.retriever import create_retriever_tool

import chromadb
chromadb.config.Settings.telemetry_enabled = False


if __name__=='__main__':
    with open('metadata.jsonl', 'r') as jsonl_file:
        json_list = list(jsonl_file)

    json_QA = []
    for json_str in json_list:
        json_data = json.loads(json_str)
        json_QA.append(json_data)

    # Usa gli stessi embeddings
    embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
    print(1)

    # Inizializza Chroma

    from langchain.schema import Document
    from langchain_community.vectorstores import Chroma

    # Prepara la lista di documenti
    docs = []
    print("orig:",len(json_QA))
    for sample in json_QA:
        print(len(docs))
        content = f"Question : {sample['Question']}\n\nFinal answer : {sample['Final answer']}"
        metadata = {"source": sample['task_id']}
        doc = Document(page_content=content, metadata=metadata)
        docs.append(doc)

    # Inizializza il vector store Chroma
    vector_store = Chroma.from_documents(
        documents=docs,
        embedding=embeddings,
        persist_directory="./chroma_db" 
    )


'''

   # Ricrea lo stesso oggetto embeddings usato nella creazione

embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")



# Carica il vector store salvato precedentemente

vector_store = Chroma(

    embedding_function=embeddings,

    persist_directory="./chroma_db"  # stesso path usato durante il salvataggio

)



# Ottieni il retriever

retriever = vector_store.as_retriever()

query = "How many more blocks (also denoted as layers) in BERT base encoder than the encoder from the architecture proposed in Attention is All You Need?"

results = retriever.invoke(query)

print(results[0].page_content) 

'''