File size: 4,293 Bytes
6fd370d
1a622af
 
 
 
 
 
 
 
 
 
6fd370d
2dfb9db
6fd370d
1a622af
 
7a50391
1a622af
7a50391
 
 
 
2dfb9db
1a622af
2dfb9db
 
7a50391
 
 
 
 
06922f2
7a50391
 
1a622af
6fd370d
7a50391
1a622af
6fd370d
 
1a622af
7a50391
 
6fd370d
7a50391
6fd370d
1a622af
7a50391
 
 
6fd370d
7a50391
6fd370d
7a50391
 
 
 
 
 
6fd370d
1a622af
7a50391
 
 
6fd370d
1a622af
7a50391
 
 
 
 
 
 
6fd370d
7a50391
 
 
1a622af
 
6fd370d
 
7a50391
 
 
 
 
6fd370d
7a50391
6fd370d
 
7a50391
6fd370d
1a622af
06922f2
7a50391
1a622af
 
6fd370d
 
06922f2
7a50391
 
 
 
06922f2
7a50391
 
 
 
 
6fd370d
7a50391
 
 
 
 
6fd370d
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
import os
import sys

# --- 1. CHROMA DB FIX FOR HUGGING FACE ---
# ChromaDB requires a newer version of sqlite3 than the one pre-installed on Linux
try:
    __import__('pysqlite3')
    sys.modules['sqlite3'] = sys.modules.pop('pysqlite3')
except ImportError:
    pass # Pass if running locally or if not available

import gradio as gr
from langchain_community.document_loaders import PyPDFLoader
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
# STABLE IMPORT (Matches langchain==0.1.20)
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma
# STABLE IMPORT
from langchain.chains import RetrievalQA
from datasets import Dataset
from ragas import evaluate
from ragas.metrics import faithfulness, answer_relevancy

# --- 2. KEY LOADER ---
api_key = os.getenv("OPENAI_API_KEY")

if api_key:
    key_status = "✅ ACTIVE (Loaded from Secrets)"
    os.environ["OPENAI_API_KEY"] = api_key
else:
    key_status = "❌ MISSING (Check Settings -> Secrets)"

def audit_rag(pdf_file, user_question):
    if not api_key:
        return "ERROR: API Key is missing.", "ERROR", "0", "0"

    if not pdf_file or not user_question:
        return "Please upload a PDF and ask a question.", "Waiting...", "0.00", "0.00"

    try:
        # Load & Split
        loader = PyPDFLoader(pdf_file.name)
        documents = loader.load()
        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
        texts = text_splitter.split_documents(documents)
        
        # RAG Engine
        embeddings = OpenAIEmbeddings(openai_api_key=api_key)
        db = Chroma.from_documents(texts, embeddings)
        retriever = db.as_retriever(search_kwargs={"k": 3})
        
        llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0, openai_api_key=api_key)
        
        qa_chain = RetrievalQA.from_chain_type(
            llm=llm, 
            chain_type="stuff", 
            retriever=retriever,
            return_source_documents=True
        )
        
        # Answer
        result = qa_chain.invoke({"query": user_question})
        generated_answer = result['result']
        source_docs = [doc.page_content for doc in result['source_documents']]
        
        # Ragas Audit
        data = {
            'question': [user_question],
            'answer': [generated_answer],
            'contexts': [source_docs],
            'ground_truth': [""] 
        }
        dataset = Dataset.from_dict(data)
        
        score = evaluate(
            dataset=dataset,
            metrics=[faithfulness, answer_relevancy],
            llm=llm,       
            embeddings=embeddings
        )
        
        audit_results = score.to_pandas()
        faith_score = audit_results.iloc[0]['faithfulness']
        relevancy_score = audit_results.iloc[0]['answer_relevancy']

        verdict = "✅ PASS" if faith_score > 0.8 else "❌ FAIL (Hallucination Detected)"
        
        return generated_answer, verdict, f"{faith_score:.2f}", f"{relevancy_score:.2f}"

    except Exception as e:
        return f"System Error: {str(e)}", "ERROR", "0", "0"

# UI
with gr.Blocks(theme=gr.themes.Soft()) as demo:
    gr.Markdown("# ⚖️ Veritas: AI Hallucination Auditor")
    gr.Markdown(f"**System Status:** {key_status}")
    gr.Markdown("Upload a document (e.g., Financial Report) and ask a question.")
    
    with gr.Row():
        with gr.Column():
            file_input = gr.File(label="Upload PDF Evidence", file_types=[".pdf"])
            question_input = gr.Textbox(label="Cross-Examination Question", placeholder="e.g., What was the net profit in Q3?")
            submit_btn = gr.Button("Run Audit", variant="primary")
        
        with gr.Column():
            answer_output = gr.Textbox(label="AI Witness Testimony (Answer)")
            with gr.Row():
                verdict_output = gr.Textbox(label="Verdict")
                faith_output = gr.Textbox(label="Faithfulness Score (0-1)")
            relevance_output = gr.Textbox(label="Relevancy Score")

    submit_btn.click(
        audit_rag, 
        inputs=[file_input, question_input], 
        outputs=[answer_output, verdict_output, faith_output, relevance_output]
    )

if __name__ == "__main__":
    demo.launch()