Spaces:
Running
Running
File size: 2,155 Bytes
4847e7d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | import hashlib
import os
import re
from urllib.parse import urlparse
import psycopg2
from dotenv import load_dotenv
from sentence_transformers import SentenceTransformer
# =====================================================
# LOAD ENV
# =====================================================
load_dotenv()
# =====================================================
# CONFIG
# =====================================================
CHUNK_SIZE = 220
DB_CONFIG = {
"host": os.getenv("SQL_DATABASE_HOST"),
"dbname": os.getenv("SQL_DATABASE"),
"user": os.getenv("SQL_USER"),
"password": os.getenv("SQL_PASSWORD"),
"port": os.getenv("SQL_DATABASE_PORT", "5432"),
"sslmode": "require"
}
# =====================================================
# GLOBALS
# =====================================================
_EMBEDDER = None
def get_embedder():
"""Lazy load the sentence transformer model."""
global _EMBEDDER
if _EMBEDDER is None:
_EMBEDDER = SentenceTransformer(
"nomic-ai/nomic-embed-text-v1",
trust_remote_code=True
)
return _EMBEDDER
# =====================================================
# DB SETUP
# =====================================================
def get_db_connection():
return psycopg2.connect(**DB_CONFIG)
# =====================================================
# UTILS
# =====================================================
def normalize_url(url):
parsed = urlparse(url)
return f"{parsed.scheme}://{parsed.netloc}{parsed.path}".rstrip("/")
def clean_text(text):
return text.replace("\x00", "").strip()
def page_hash(text):
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def chunk_hash(text):
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def chunk_text(text, size=200, overlap=50):
words = text.split()
step = size - overlap
for i in range(0, len(words), step):
yield " ".join(words[i:i + size])
def extract_keywords(question):
words = re.findall(r'\b[a-zA-Z]{3,}\b', question.lower())
return list(set(words)) |