File size: 4,293 Bytes
6fd370d 1a622af 6fd370d 2dfb9db 6fd370d 1a622af 7a50391 1a622af 7a50391 2dfb9db 1a622af 2dfb9db 7a50391 06922f2 7a50391 1a622af 6fd370d 7a50391 1a622af 6fd370d 1a622af 7a50391 6fd370d 7a50391 6fd370d 1a622af 7a50391 6fd370d 7a50391 6fd370d 7a50391 6fd370d 1a622af 7a50391 6fd370d 1a622af 7a50391 6fd370d 7a50391 1a622af 6fd370d 7a50391 6fd370d 7a50391 6fd370d 7a50391 6fd370d 1a622af 06922f2 7a50391 1a622af 6fd370d 06922f2 7a50391 06922f2 7a50391 6fd370d 7a50391 6fd370d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 |
import os
import sys
# --- 1. CHROMA DB FIX FOR HUGGING FACE ---
# ChromaDB requires a newer version of sqlite3 than the one pre-installed on Linux
try:
__import__('pysqlite3')
sys.modules['sqlite3'] = sys.modules.pop('pysqlite3')
except ImportError:
pass # Pass if running locally or if not available
import gradio as gr
from langchain_community.document_loaders import PyPDFLoader
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
# STABLE IMPORT (Matches langchain==0.1.20)
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma
# STABLE IMPORT
from langchain.chains import RetrievalQA
from datasets import Dataset
from ragas import evaluate
from ragas.metrics import faithfulness, answer_relevancy
# --- 2. KEY LOADER ---
api_key = os.getenv("OPENAI_API_KEY")
if api_key:
key_status = "✅ ACTIVE (Loaded from Secrets)"
os.environ["OPENAI_API_KEY"] = api_key
else:
key_status = "❌ MISSING (Check Settings -> Secrets)"
def audit_rag(pdf_file, user_question):
if not api_key:
return "ERROR: API Key is missing.", "ERROR", "0", "0"
if not pdf_file or not user_question:
return "Please upload a PDF and ask a question.", "Waiting...", "0.00", "0.00"
try:
# Load & Split
loader = PyPDFLoader(pdf_file.name)
documents = loader.load()
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
texts = text_splitter.split_documents(documents)
# RAG Engine
embeddings = OpenAIEmbeddings(openai_api_key=api_key)
db = Chroma.from_documents(texts, embeddings)
retriever = db.as_retriever(search_kwargs={"k": 3})
llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0, openai_api_key=api_key)
qa_chain = RetrievalQA.from_chain_type(
llm=llm,
chain_type="stuff",
retriever=retriever,
return_source_documents=True
)
# Answer
result = qa_chain.invoke({"query": user_question})
generated_answer = result['result']
source_docs = [doc.page_content for doc in result['source_documents']]
# Ragas Audit
data = {
'question': [user_question],
'answer': [generated_answer],
'contexts': [source_docs],
'ground_truth': [""]
}
dataset = Dataset.from_dict(data)
score = evaluate(
dataset=dataset,
metrics=[faithfulness, answer_relevancy],
llm=llm,
embeddings=embeddings
)
audit_results = score.to_pandas()
faith_score = audit_results.iloc[0]['faithfulness']
relevancy_score = audit_results.iloc[0]['answer_relevancy']
verdict = "✅ PASS" if faith_score > 0.8 else "❌ FAIL (Hallucination Detected)"
return generated_answer, verdict, f"{faith_score:.2f}", f"{relevancy_score:.2f}"
except Exception as e:
return f"System Error: {str(e)}", "ERROR", "0", "0"
# UI
with gr.Blocks(theme=gr.themes.Soft()) as demo:
gr.Markdown("# ⚖️ Veritas: AI Hallucination Auditor")
gr.Markdown(f"**System Status:** {key_status}")
gr.Markdown("Upload a document (e.g., Financial Report) and ask a question.")
with gr.Row():
with gr.Column():
file_input = gr.File(label="Upload PDF Evidence", file_types=[".pdf"])
question_input = gr.Textbox(label="Cross-Examination Question", placeholder="e.g., What was the net profit in Q3?")
submit_btn = gr.Button("Run Audit", variant="primary")
with gr.Column():
answer_output = gr.Textbox(label="AI Witness Testimony (Answer)")
with gr.Row():
verdict_output = gr.Textbox(label="Verdict")
faith_output = gr.Textbox(label="Faithfulness Score (0-1)")
relevance_output = gr.Textbox(label="Relevancy Score")
submit_btn.click(
audit_rag,
inputs=[file_input, question_input],
outputs=[answer_output, verdict_output, faith_output, relevance_output]
)
if __name__ == "__main__":
demo.launch() |