|
|
import os |
|
|
import sys |
|
|
|
|
|
|
|
|
|
|
|
try: |
|
|
__import__('pysqlite3') |
|
|
sys.modules['sqlite3'] = sys.modules.pop('pysqlite3') |
|
|
except ImportError: |
|
|
pass |
|
|
|
|
|
import gradio as gr |
|
|
from langchain_community.document_loaders import PyPDFLoader |
|
|
from langchain_openai import ChatOpenAI, OpenAIEmbeddings |
|
|
|
|
|
from langchain.text_splitter import RecursiveCharacterTextSplitter |
|
|
from langchain_community.vectorstores import Chroma |
|
|
|
|
|
from langchain.chains import RetrievalQA |
|
|
from datasets import Dataset |
|
|
from ragas import evaluate |
|
|
from ragas.metrics import faithfulness, answer_relevancy |
|
|
|
|
|
|
|
|
api_key = os.getenv("OPENAI_API_KEY") |
|
|
|
|
|
if api_key: |
|
|
key_status = "β
ACTIVE (Loaded from Secrets)" |
|
|
os.environ["OPENAI_API_KEY"] = api_key |
|
|
else: |
|
|
key_status = "β MISSING (Check Settings -> Secrets)" |
|
|
|
|
|
def audit_rag(pdf_file, user_question): |
|
|
if not api_key: |
|
|
return "ERROR: API Key is missing.", "ERROR", "0", "0" |
|
|
|
|
|
if not pdf_file or not user_question: |
|
|
return "Please upload a PDF and ask a question.", "Waiting...", "0.00", "0.00" |
|
|
|
|
|
try: |
|
|
|
|
|
loader = PyPDFLoader(pdf_file.name) |
|
|
documents = loader.load() |
|
|
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) |
|
|
texts = text_splitter.split_documents(documents) |
|
|
|
|
|
|
|
|
embeddings = OpenAIEmbeddings(openai_api_key=api_key) |
|
|
db = Chroma.from_documents(texts, embeddings) |
|
|
retriever = db.as_retriever(search_kwargs={"k": 3}) |
|
|
|
|
|
llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0, openai_api_key=api_key) |
|
|
|
|
|
qa_chain = RetrievalQA.from_chain_type( |
|
|
llm=llm, |
|
|
chain_type="stuff", |
|
|
retriever=retriever, |
|
|
return_source_documents=True |
|
|
) |
|
|
|
|
|
|
|
|
result = qa_chain.invoke({"query": user_question}) |
|
|
generated_answer = result['result'] |
|
|
source_docs = [doc.page_content for doc in result['source_documents']] |
|
|
|
|
|
|
|
|
data = { |
|
|
'question': [user_question], |
|
|
'answer': [generated_answer], |
|
|
'contexts': [source_docs], |
|
|
'ground_truth': [""] |
|
|
} |
|
|
dataset = Dataset.from_dict(data) |
|
|
|
|
|
score = evaluate( |
|
|
dataset=dataset, |
|
|
metrics=[faithfulness, answer_relevancy], |
|
|
llm=llm, |
|
|
embeddings=embeddings |
|
|
) |
|
|
|
|
|
audit_results = score.to_pandas() |
|
|
faith_score = audit_results.iloc[0]['faithfulness'] |
|
|
relevancy_score = audit_results.iloc[0]['answer_relevancy'] |
|
|
|
|
|
verdict = "β
PASS" if faith_score > 0.8 else "β FAIL (Hallucination Detected)" |
|
|
|
|
|
return generated_answer, verdict, f"{faith_score:.2f}", f"{relevancy_score:.2f}" |
|
|
|
|
|
except Exception as e: |
|
|
return f"System Error: {str(e)}", "ERROR", "0", "0" |
|
|
|
|
|
|
|
|
with gr.Blocks(theme=gr.themes.Soft()) as demo: |
|
|
gr.Markdown("# βοΈ Veritas: AI Hallucination Auditor") |
|
|
gr.Markdown(f"**System Status:** {key_status}") |
|
|
gr.Markdown("Upload a document (e.g., Financial Report) and ask a question.") |
|
|
|
|
|
with gr.Row(): |
|
|
with gr.Column(): |
|
|
file_input = gr.File(label="Upload PDF Evidence", file_types=[".pdf"]) |
|
|
question_input = gr.Textbox(label="Cross-Examination Question", placeholder="e.g., What was the net profit in Q3?") |
|
|
submit_btn = gr.Button("Run Audit", variant="primary") |
|
|
|
|
|
with gr.Column(): |
|
|
answer_output = gr.Textbox(label="AI Witness Testimony (Answer)") |
|
|
with gr.Row(): |
|
|
verdict_output = gr.Textbox(label="Verdict") |
|
|
faith_output = gr.Textbox(label="Faithfulness Score (0-1)") |
|
|
relevance_output = gr.Textbox(label="Relevancy Score") |
|
|
|
|
|
submit_btn.click( |
|
|
audit_rag, |
|
|
inputs=[file_input, question_input], |
|
|
outputs=[answer_output, verdict_output, faith_output, relevance_output] |
|
|
) |
|
|
|
|
|
if __name__ == "__main__": |
|
|
demo.launch() |