Download app/utils/rag.py from rk147/secforge: direct link, hf CLI and curl.
- Browser
- Download file 4.81 kB
-
https://huggingface.co/spaces/rk147/secforge/resolve/main/app/utils/rag.py
- Command line
-
hf download hf://spaces/rk147/secforge/app/utils/rag.py
-
curl -L -o rag.py https://huggingface.co/spaces/rk147/secforge/resolve/main/app/utils/rag.py
4.81 kB
| """RAG utilities — persistent ChromaDB with graceful fallback.""" | |
| import os | |
| import uuid | |
| import streamlit as st | |
| from sentence_transformers import SentenceTransformer | |
| import chromadb | |
| # Storage directory for ChromaDB (persistent across calls, but not across Space restart) | |
| CHROMA_PERSIST_DIR = "/tmp/secforge_chroma" | |
| os.makedirs(CHROMA_PERSIST_DIR, exist_ok=True) | |
| _chroma_client = None | |
| _embedding_model_failed = False | |
| _rag_enabled = True # global flag to disable RAG after any fatal error | |
| def _get_client(): | |
| global _chroma_client | |
| if _chroma_client is None: | |
| try: | |
| # Use persistent client instead of ephemeral | |
| _chroma_client = chromadb.PersistentClient(path=CHROMA_PERSIST_DIR) | |
| except Exception as e: | |
| print(f"[RAG FATAL] Could not create ChromaDB client: {e}") | |
| global _rag_enabled | |
| _rag_enabled = False | |
| return None | |
| return _chroma_client | |
| def get_embedding_model(): | |
| global _embedding_model_failed | |
| if _embedding_model_failed: | |
| return None | |
| try: | |
| return SentenceTransformer("all-MiniLM-L6-v2") | |
| except Exception as e: | |
| print(f"[RAG WARNING] Failed to load embedding model: {e}") | |
| print("[RAG WARNING] File uploads will work but semantic search will be disabled") | |
| _embedding_model_failed = True | |
| return None | |
| def add_file(session_id: str, filename: str, text: str): | |
| if not _rag_enabled: | |
| return | |
| client = _get_client() | |
| if client is None: | |
| return | |
| model = get_embedding_model() | |
| if model is None: | |
| return | |
| collection_name = f"s{session_id[:32]}" | |
| try: | |
| # Get or create collection - use get_or_create_collection to avoid race conditions | |
| collection = client.get_or_create_collection(name=collection_name) | |
| except Exception as e: | |
| print(f"[RAG ERROR] Failed to get/create collection: {e}") | |
| return | |
| # Remove existing chunks for this file | |
| try: | |
| existing = collection.get(where={"source": filename}) | |
| if existing["ids"]: | |
| collection.delete(ids=existing["ids"]) | |
| except Exception: | |
| pass | |
| chunks = [line.strip() for line in text.split("\n") if line.strip()] | |
| if not chunks: | |
| return | |
| try: | |
| embeddings = model.encode(chunks).tolist() | |
| ids = [str(uuid.uuid4()) for _ in chunks] | |
| metadatas = [{"source": filename} for _ in chunks] | |
| collection.add(embeddings=embeddings, documents=chunks, ids=ids, metadatas=metadatas) | |
| except Exception as e: | |
| print(f"[RAG ERROR] Failed to add file {filename}: {e}") | |
| def remove_file(session_id: str, filename: str): | |
| if not _rag_enabled: | |
| return | |
| client = _get_client() | |
| if client is None: | |
| return | |
| collection_name = f"s{session_id[:32]}" | |
| try: | |
| collection = client.get_collection(name=collection_name) | |
| existing = collection.get(where={"source": filename}) | |
| if existing["ids"]: | |
| collection.delete(ids=existing["ids"]) | |
| except Exception: | |
| pass | |
| def clear_session(session_id: str): | |
| if not _rag_enabled: | |
| return | |
| client = _get_client() | |
| if client is None: | |
| return | |
| collection_name = f"s{session_id[:32]}" | |
| try: | |
| client.delete_collection(name=collection_name) | |
| except Exception: | |
| pass | |
| def query_context(session_id: str, query: str, n_results: int = 8) -> str: | |
| if not _rag_enabled: | |
| return "" | |
| client = _get_client() | |
| if client is None: | |
| return "" | |
| model = get_embedding_model() | |
| if model is None: | |
| return "" | |
| collection_name = f"s{session_id[:32]}" | |
| try: | |
| collection = client.get_collection(name=collection_name) | |
| except Exception: | |
| # Collection does not exist – no context | |
| return "" | |
| try: | |
| count = collection.count() | |
| if count == 0: | |
| return "" | |
| actual_n = min(n_results, count) | |
| query_embedding = model.encode([query]).tolist() | |
| results = collection.query( | |
| query_embeddings=query_embedding, | |
| n_results=actual_n, | |
| include=["documents", "metadatas"], | |
| ) | |
| docs = results["documents"][0] | |
| metas = results["metadatas"][0] | |
| grouped = {} | |
| for doc, meta in zip(docs, metas): | |
| src = meta.get("source", "unknown") | |
| grouped.setdefault(src, []).append(doc) | |
| parts = [] | |
| for src, lines in grouped.items(): | |
| parts.append(f"# from {src}\n" + "\n".join(lines)) | |
| return "\n\n".join(parts) | |
| except Exception as e: | |
| print(f"[RAG ERROR] query failed: {e}") | |
| return "" | |
| def add_document(collection_name: str, text: str): | |
| add_file(collection_name, "linked_file", text) |