Spaces:
Sleeping
Sleeping
File size: 2,074 Bytes
ecb9ae4 c50b839 ecb9ae4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 | import os
from langchain_text_splitters import MarkdownHeaderTextSplitter, RecursiveCharacterTextSplitter
from src.database import get_embeddings
from langchain_postgres import PGVector
from dotenv import load_dotenv
load_dotenv()
def load_and_split_data(file_path='data/reglement.md'):
"""Lit le fichier Markdown et le découpe sémantiquement en chunks."""
print(f"Lecture du fichier : {file_path}")
with open(file_path, "r", encoding="utf-8") as f:
markdown_document = f.read()
headers_to_split_on = [
("#", "Sujet_Principal"),
("##", "Type_Regle"),
]
markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
md_header_splits = markdown_splitter.split_text(markdown_document)
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=700,
chunk_overlap=200,
separators=["\n\n", "\n", ".", " ", ""]
)
final_chunks = text_splitter.split_documents(md_header_splits)
print(f"Découpage terminé : {len(final_chunks)} chunks générés.")
return final_chunks
def build_index(chunks, db_url: str, collection_name: str):
"""Transforme les chunks en vecteurs avec bge-m3 et les stocke dans PostgreSQL."""
print("Chargement du modèle BAAI/bge-m3...")
embeddings = get_embeddings()
print(f"Connexion à la base et création de l'index vectoriel...")
vector_store = PGVector.from_documents(
embedding=embeddings,
documents=chunks,
collection_name=collection_name,
connection=db_url,
use_jsonb=True,
)
print("Indexation réussie dans la base de données PostgreSQL !")
return vector_store
if __name__ == "__main__":
FILE_PATH = "data/reglement.md"
COLLECTION_NAME = "rh_cgi_vectors"
DATABASE_URL = os.getenv('DATABASE_URL')
if not DATABASE_URL:
raise ValueError("Erreur : La variable DATABASE_URL est introuvable.")
documents = load_and_split_data(FILE_PATH)
vector_store = build_index(documents, DATABASE_URL, COLLECTION_NAME) |