Add application file

README.md

@@ -1,12 +1,41 @@
----
-title: Test
-emoji: 🐢
-colorFrom: gray
-colorTo: yellow
-sdk: streamlit
-sdk_version: 1.21.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# auto_evaluator
+
+### Aim
+In this lesson we want to create a app which thake some data and from that data we provide some questions and answer. Then we test that answer through LLM and grade them.
+
+Context
+
+Document Question-Answering is a popular LLM use-case. LangChain makes it easy to assemble LLM components (e.g., models and retrievers) into chains that support question-answering: input documents are split into chunks and stored in a retriever, relevant chunks are retrieved given a user question and passed to an LLM for synthesis into an answer.
+
+Challenge
+
+The quality of QA systems can vary considerably; for example, we have seen cases of hallucination and poor answer quality due specific parameter settings. But, it is not always obvious to (1) evaluate the answer quality in a systematic way and (2) use this evaluation to guide improved QA chain settings (e.g., chunk size) or components (e.g., model or retriever choice).
+
+App overview
+
+This app aims to address the above limitations. Recent work from Anthropic has used model-written evaluation sets. OpenAI and others have shown that model-graded evaluation is an effective way to evaluate models. This app combines both of these ideas into a single workspace, auto-generating a QA test set and auto-grading the result of the specified QA chain.
+
+![image](https://github.com/DeepakJaiz/auto_evaluator/assets/120568685/d9a2aaac-e8c1-4836-b6d8-7121ad797ccf)
+
+### Reference
+https://github.com/langchain-ai/auto-evaluator/blob/main/README.md
+
+### Prerequist
+`You required Phython in your system fyou can follow this (https://www.python.org)`<br/>
+`OpenAi API key required to get you can follow this link (https://platform.openai.com/account/api-keys)`<br/>
+`Create a account on huggingface https://huggingface.co/login`<br/>
+
+### Modules required
+All the requirment are define in requirment.text file.
+### How to run
+1. Open the huggingface after login.
+2. Go to the space button and then create new space.
+3. For creating new space enter the name of space as you want, then select "streamlit" from select the space sdk. Then make your space public or private as you want.
+4. After that click the create space.
+5. Then upload all the above file in your space.
+6. Then you can see your application by clicking the app button.
+
+### How to use app
+1. In left side enter your open API Key.
+2. Upload you source data in pdf and text format.
+3. Upload your test question answer in JSON format just below it.

app.py

@@ -0,0 +1,491 @@
+import os
+import json
+import time
+from typing import List
+import faiss
+import pypdf
+import random
+import itertools
+import text_utils
+import pandas as pd
+import altair as alt
+import streamlit as st
+from io import StringIO
+from llama_index import Document
+from langchain.llms import Anthropic
+from langchain import HuggingFaceHub
+from langchain.chains import RetrievalQA
+from langchain.vectorstores import FAISS
+from llama_index import LangchainEmbedding
+from langchain.chat_models import ChatOpenAI
+from langchain.retrievers import SVMRetriever
+from langchain.chains import QAGenerationChain
+from langchain.retrievers import TFIDFRetriever
+from langchain.evaluation.qa import QAEvalChain
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.embeddings.openai import OpenAIEmbeddings
+from gpt_index import LLMPredictor, ServiceContext, GPTFaissIndex
+from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter
+from text_utils import GRADE_DOCS_PROMPT, GRADE_ANSWER_PROMPT, GRADE_DOCS_PROMPT_FAST, GRADE_ANSWER_PROMPT_FAST, GRADE_ANSWER_PROMPT_BIAS_CHECK, GRADE_ANSWER_PROMPT_OPENAI
+
+# Keep dataframe in memory to accumulate experimental results
+if "existing_df" not in st.session_state:
+    summary = pd.DataFrame(columns=['chunk_chars',
+                                    'overlap',
+                                    'split',
+                                    'model',
+                                    'retriever',
+                                    'embedding',
+                                    'num_neighbors',
+                                    'Latency',
+                                    'Retrieval score',
+                                    'Answer score'])
+    st.session_state.existing_df = summary
+else:
+    summary = st.session_state.existing_df
+
+
+@st.cache_data
+def load_docs(files: List) -> str:
+    """
+    Load docs from files
+    @param files: list of files to load
+    @return: string of all docs concatenated
+    """
+
+    st.info("`Reading doc ...`")
+    all_text = ""
+    for file_path in files:
+        file_extension = os.path.splitext(file_path.name)[1]
+        if file_extension == ".pdf":
+            pdf_reader = pypdf.PdfReader(file_path)
+            file_content = ""
+            for page in pdf_reader.pages:
+                file_content += page.extract_text()
+            file_content = text_utils.clean_pdf_text(file_content)
+            all_text += file_content
+        elif file_extension == ".txt":
+            stringio = StringIO(file_path.getvalue().decode("utf-8"))
+            file_content = stringio.read()
+            all_text += file_content
+        else:
+            st.warning('Please provide txt or pdf.', icon="⚠️")
+    return all_text
+
+
+@st.cache_data
+def generate_eval(text: str, num_questions: int, chunk: int):
+    """
+    Generate eval set
+    @param text: text to generate eval set from
+    @param num_questions: number of questions to generate
+    @param chunk: chunk size to draw question from in the doc
+    @return: eval set as JSON list
+    """
+    st.info("`Generating eval set ...`")
+    n = len(text)
+    starting_indices = [random.randint(0, n - chunk) for _ in range(num_questions)]
+    sub_sequences = [text[i:i + chunk] for i in starting_indices]
+    chain = QAGenerationChain.from_llm(ChatOpenAI(temperature=0))
+    eval_set = []
+    for i, b in enumerate(sub_sequences):
+        try:
+            qa = chain.run(b)
+            eval_set.append(qa)
+        except:
+            st.warning('Error generating question %s.' % str(i + 1), icon="⚠️")
+    eval_set_full = list(itertools.chain.from_iterable(eval_set))
+    return eval_set_full
+
+
+@st.cache_resource
+def split_texts(text, chunk_size: int, overlap, split_method: str):
+    """
+    Split text into chunks
+    @param text: text to split
+    @param chunk_size:
+    @param overlap:
+    @param split_method:
+    @return: list of str splits
+    """
+    st.info("`Splitting doc ...`")
+    if split_method == "RecursiveTextSplitter":
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size,
+                                                       chunk_overlap=overlap)
+    elif split_method == "CharacterTextSplitter":
+        text_splitter = CharacterTextSplitter(separator=" ",
+                                              chunk_size=chunk_size,
+                                              chunk_overlap=overlap)
+    else:
+        st.warning("`Split method not recognized. Using RecursiveCharacterTextSplitter`", icon="⚠️")
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size,
+                                                       chunk_overlap=overlap)
+
+    split_text = text_splitter.split_text(text)
+    return split_text
+
+
+@st.cache_resource
+def make_llm(model_version: str):
+    """
+    Make LLM from model version
+    @param model_version: model_version
+    @return: LLN
+    """
+    if (model_version == "gpt-3.5-turbo") or (model_version == "gpt-4"):
+        chosen_model = ChatOpenAI(model_name=model_version, temperature=0)
+    elif model_version == "anthropic":
+        chosen_model = Anthropic(temperature=0)
+    elif model_version == "flan-t5-xl":
+        chosen_model = HuggingFaceHub(repo_id="google/flan-t5-xl",model_kwargs={"temperature":0,"max_length":64})
+    else:
+        st.warning("`Model version not recognized. Using gpt-3.5-turbo`", icon="⚠️")
+        chosen_model = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
+    return chosen_model
+
+@st.cache_resource
+def make_retriever(splits, retriever_type, embedding_type, num_neighbors, _llm):
+    """
+    Make document retriever
+    @param splits: list of str splits
+    @param retriever_type: retriever type
+    @param embedding_type: embedding type
+    @param num_neighbors: number of neighbors for retrieval
+    @param _llm: model
+    @return: retriever
+    """
+    st.info("`Making retriever ...`")
+    # Set embeddings
+    if embedding_type == "OpenAI":
+        embedding = OpenAIEmbeddings()
+    elif embedding_type == "HuggingFace":
+        embedding = HuggingFaceEmbeddings()
+    else:
+        st.warning("`Embedding type not recognized. Using OpenAI`", icon="⚠️")
+        embedding = OpenAIEmbeddings()
+
+    # Select retriever
+    if retriever_type == "similarity-search":
+        try:
+            vector_store = FAISS.from_texts(splits, embedding)
+        except ValueError:
+            st.warning("`Error using OpenAI embeddings (disallowed TikToken token in the text). Using HuggingFace.`",
+                       icon="⚠️")
+            vector_store = FAISS.from_texts(splits, HuggingFaceEmbeddings())
+        retriever_obj = vector_store.as_retriever(k=num_neighbors)
+    elif retriever_type == "SVM":
+        retriever_obj = SVMRetriever.from_texts(splits, embedding)
+    elif retriever_type == "TF-IDF":
+        retriever_obj = TFIDFRetriever.from_texts(splits)
+    elif retriever_type == "Llama-Index":
+        documents = [Document(t, LangchainEmbedding(embedding)) for t in splits]
+        llm_predictor = LLMPredictor(llm)
+        context = ServiceContext.from_defaults(chunk_size_limit=512, llm_predictor=llm_predictor)
+        d = 1536
+        faiss_index = faiss.IndexFlatL2(d)
+        retriever_obj = GPTFaissIndex.from_documents(documents, faiss_index=faiss_index, service_context=context)
+    else:
+        st.warning("`Retriever type not recognized. Using SVM`", icon="⚠️")
+        retriever_obj = SVMRetriever.from_texts(splits, embedding)
+    return retriever_obj
+
+
+def make_chain(llm, retriever, retriever_type: str) -> RetrievalQA:
+    """
+    Make chain
+    @param llm: model
+    @param retriever: retriever
+    @param retriever_type: retriever type
+    @return: chain (or return retriever for Llama-Index)
+    """
+    st.info("`Making chain ...`")
+    if retriever_type == "Llama-Index":
+        qa = retriever
+    else:
+        qa = RetrievalQA.from_chain_type(llm,
+                                         chain_type="stuff",
+                                         retriever=retriever,
+                                         input_key="question")
+    return qa
+
+
+def grade_model_answer(predicted_dataset: List, predictions: List, grade_answer_prompt: str) -> List:
+    """
+    Grades the distilled answer based on ground truth and model predictions.
+    @param predicted_dataset: A list of dictionaries containing ground truth questions and answers.
+    @param predictions: A list of dictionaries containing model predictions for the questions.
+    @param grade_answer_prompt: The prompt level for the grading. Either "Fast" or "Full".
+    @return: A list of scores for the distilled answers.
+    """
+    # Grade the distilled answer
+    st.info("`Grading model answer ...`")
+    # Set the grading prompt based on the grade_answer_prompt parameter
+    if grade_answer_prompt == "Fast":
+        prompt = GRADE_ANSWER_PROMPT_FAST
+    elif grade_answer_prompt == "Descriptive w/ bias check":
+        prompt = GRADE_ANSWER_PROMPT_BIAS_CHECK
+    elif grade_answer_prompt == "OpenAI grading prompt":
+        prompt = GRADE_ANSWER_PROMPT_OPENAI
+    else:
+        prompt = GRADE_ANSWER_PROMPT
+
+    # Create an evaluation chain
+    eval_chain = QAEvalChain.from_llm(
+        llm=ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0),
+        prompt=prompt
+    )
+
+    # Evaluate the predictions and ground truth using the evaluation chain
+    graded_outputs = eval_chain.evaluate(
+        predicted_dataset,
+        predictions,
+        question_key="question",
+        prediction_key="result"
+    )
+
+    return graded_outputs
+
+
+def grade_model_retrieval(gt_dataset: List, predictions: List, grade_docs_prompt: str):
+    """
+    Grades the relevance of retrieved documents based on ground truth and model predictions.
+    @param gt_dataset: list of dictionaries containing ground truth questions and answers.
+    @param predictions: list of dictionaries containing model predictions for the questions
+    @param grade_docs_prompt: prompt level for the grading. Either "Fast" or "Full"
+    @return: list of scores for the retrieved documents.
+    """
+    # Grade the docs retrieval
+    st.info("`Grading relevance of retrieved docs ...`")
+
+    # Set the grading prompt based on the grade_docs_prompt parameter
+    prompt = GRADE_DOCS_PROMPT_FAST if grade_docs_prompt == "Fast" else GRADE_DOCS_PROMPT
+
+    # Create an evaluation chain
+    eval_chain = QAEvalChain.from_llm(
+        llm=ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0),
+        prompt=prompt
+    )
+
+    # Evaluate the predictions and ground truth using the evaluation chain
+    graded_outputs = eval_chain.evaluate(
+        gt_dataset,
+        predictions,
+        question_key="question",
+        prediction_key="result"
+    )
+    return graded_outputs
+
+
+def run_evaluation(chain, retriever, eval_set, grade_prompt, retriever_type, num_neighbors):
+    """
+    Runs evaluation on a model's performance on a given evaluation dataset.
+    @param chain: Model chain used for answering questions
+    @param retriever:  Document retriever used for retrieving relevant documents
+    @param eval_set: List of dictionaries containing questions and corresponding ground truth answers
+    @param grade_prompt: String prompt used for grading model's performance
+    @param retriever_type: String specifying the type of retriever used
+    @param num_neighbors: Number of neighbors to retrieve using the retriever
+    @return: A tuple of four items:
+    - answers_grade: A dictionary containing scores for the model's answers.
+    - retrieval_grade: A dictionary containing scores for the model's document retrieval.
+    - latencies_list: A list of latencies in seconds for each question answered.
+    - predictions_list: A list of dictionaries containing the model's predicted answers and relevant documents for each question.
+    """
+    st.info("`Running evaluation ...`")
+    predictions_list = []
+    retrieved_docs = []
+    gt_dataset = []
+    latencies_list = []
+
+    for data in eval_set:
+
+        # Get answer and log latency
+        start_time = time.time()
+        if retriever_type != "Llama-Index":
+            predictions_list.append(chain(data))
+        elif retriever_type == "Llama-Index":
+            answer = chain.query(data["question"], similarity_top_k=num_neighbors, response_mode="tree_summarize",
+                                 use_async=True)
+            predictions_list.append({"question": data["question"], "answer": data["answer"], "result": answer.response})
+        gt_dataset.append(data)
+        end_time = time.time()
+        elapsed_time = end_time - start_time
+        latencies_list.append(elapsed_time)
+
+        # Retrieve docs
+        retrieved_doc_text = ""
+        if retriever_type == "Llama-Index":
+            for i, doc in enumerate(answer.source_nodes):
+                retrieved_doc_text += "Doc %s: " % str(i + 1) + doc.node.text + " "
+
+        else:
+            docs = retriever.get_relevant_documents(data["question"])
+            for i, doc in enumerate(docs):
+                retrieved_doc_text += "Doc %s: " % str(i + 1) + doc.page_content + " "
+
+        retrieved = {"question": data["question"], "answer": data["answer"], "result": retrieved_doc_text}
+        retrieved_docs.append(retrieved)
+
+    # Grade
+    answers_grade = grade_model_answer(gt_dataset, predictions_list, grade_prompt)
+    retrieval_grade = grade_model_retrieval(gt_dataset, retrieved_docs, grade_prompt)
+    return answers_grade, retrieval_grade, latencies_list, predictions_list
+
+
+# Auth
+st.sidebar.image("img/diagnostic.jpg")
+
+oai_api_key = st.sidebar.text_input("`OpenAI API Key:`", type="password")
+ant_api_key = st.sidebar.text_input("`(Optional) Anthropic API Key:`", type="password")
+hf_api_key = st.sidebar.text_input("`(Optional) HuggingFace API Token:`", type="password")
+
+with st.sidebar.form("user_input"):
+
+    num_eval_questions = st.select_slider("`Number of eval questions`",
+                                          options=[1, 5, 10, 15, 20], value=5)
+
+    chunk_chars = st.select_slider("`Choose chunk size for splitting`",
+                                   options=[500, 750, 1000, 1500, 2000], value=1000)
+
+    overlap = st.select_slider("`Choose overlap for splitting`",
+                               options=[0, 50, 100, 150, 200], value=100)
+
+    split_method = st.radio("`Split method`",
+                            ("RecursiveTextSplitter",
+                             "CharacterTextSplitter"),
+                            index=0)
+
+    model = st.radio("`Choose model`",
+                     ("gpt-3.5-turbo",
+                      "gpt-4",
+                      "anthropic"),
+                      # Error raised by inference API: Model google/flan-t5-xl time out
+                      #"flan-t5-xl"),
+                     index=0)
+
+    retriever_type = st.radio("`Choose retriever`",
+                              ("TF-IDF",
+                               "SVM",
+                               "Llama-Index",
+                               "similarity-search"),
+                              index=3)
+
+    num_neighbors = st.select_slider("`Choose # chunks to retrieve`",
+                                     options=[3, 4, 5, 6, 7, 8])
+
+    embeddings = st.radio("`Choose embeddings`",
+                          ("HuggingFace",
+                           "OpenAI"),
+                          index=1)
+
+    grade_prompt = st.radio("`Grading style prompt`",
+                            ("Fast",
+                             "Descriptive",
+                             "Descriptive w/ bias check",
+                             "OpenAI grading prompt"),
+                            index=0)
+
+    submitted = st.form_submit_button("Submit evaluation")
+
+#st.sidebar.write("`By:` [@RLanceMartin](https://twitter.com/RLanceMartin)")
+
+# App
+st.header("`Auto-evaluator`")
+st.info(
+    "`I am an evaluation tool for question-answering built on LangChain. Given documents, I will auto-generate a question-answer eval "
+    "set and evaluate using the selected chain settings. Experiments with different configurations are logged. "
+    "Optionally, provide your own eval set (as a JSON, see docs/karpathy-pod-eval.json for an example). If you don't have acess to GPT-4 or Anthropic, you can use our free hosted app here: https://autoevaluator.langchain.com/`")
+
+with st.form(key='file_inputs'):
+    uploaded_file = st.file_uploader("`Please upload a file to evaluate (.txt or .pdf):` ",
+                                     type=['pdf', 'txt'],
+                                     accept_multiple_files=True)
+
+    uploaded_eval_set = st.file_uploader("`[Optional] Please upload eval set (.json):` ",
+                                         type=['json'],
+                                         accept_multiple_files=False)
+
+    submitted = st.form_submit_button("Submit files")
+
+if uploaded_file and oai_api_key:
+
+    os.environ["OPENAI_API_KEY"] = oai_api_key
+    os.environ["ANTHROPIC_API_KEY"] = ant_api_key
+    os.environ["HUGGINGFACEHUB_API_TOKEN"] = hf_api_key
+
+    # Load docs
+    text = load_docs(uploaded_file)
+    # Generate num_eval_questions questions, each from context of 3k chars randomly selected
+    if not uploaded_eval_set:
+        eval_set = generate_eval(text, num_eval_questions, 1000)
+    else:
+        eval_set = json.loads(uploaded_eval_set.read())
+    # Split text
+    splits = split_texts(text, chunk_chars, overlap, split_method)
+    # Make LLM
+    llm = make_llm(model)
+    # Make vector DB
+    retriever = make_retriever(splits, retriever_type, embeddings, num_neighbors, llm)
+    # Make chain
+    qa_chain = make_chain(llm, retriever, retriever_type)
+    # Grade model
+    graded_answers, graded_retrieval, latency, predictions = run_evaluation(qa_chain, retriever, eval_set, grade_prompt,
+                                                                      retriever_type, num_neighbors)
+
+    # Assemble outputs
+    d = pd.DataFrame(predictions)
+    d['answer score'] = [g['text'] for g in graded_answers]
+    d['docs score'] = [g['text'] for g in graded_retrieval]
+    d['latency'] = latency
+
+    # Summary statistics
+    mean_latency = d['latency'].mean()
+    correct_answer_count = len([text for text in d['answer score'] if "INCORRECT" not in text])
+    correct_docs_count = len([text for text in d['docs score'] if "Context is relevant: True" in text])
+    percentage_answer = (correct_answer_count / len(graded_answers)) * 100
+    percentage_docs = (correct_docs_count / len(graded_retrieval)) * 100
+
+    st.subheader("`Run Results`")
+    st.info(
+        "`I will grade the chain based on: 1/ the relevance of the retrived documents relative to the question and 2/ "
+        "the summarized answer relative to the ground truth answer. You can see (and change) to prompts used for "
+        "grading in text_utils`")
+    st.dataframe(data=d, use_container_width=True)
+
+    # Accumulate results
+    st.subheader("`Aggregate Results`")
+    st.info(
+        "`Retrieval and answer scores are percentage of retrived documents deemed relevant by the LLM grader ("
+        "relative to the question) and percentage of summarized answers deemed relevant (relative to ground truth "
+        "answer), respectively. The size of point correponds to the latency (in seconds) of retrieval + answer "
+        "summarization (larger circle = slower).`")
+    new_row = pd.DataFrame({'chunk_chars': [chunk_chars],
+                            'overlap': [overlap],
+                            'split': [split_method],
+                            'model': [model],
+                            'retriever': [retriever_type],
+                            'embedding': [embeddings],
+                            'num_neighbors': [num_neighbors],
+                            'Latency': [mean_latency],
+                            'Retrieval score': [percentage_docs],
+                            'Answer score': [percentage_answer]})
+    summary = pd.concat([summary, new_row], ignore_index=True)
+    st.dataframe(data=summary, use_container_width=True)
+    st.session_state.existing_df = summary
+
+    # Dataframe for visualization
+    show = summary.reset_index().copy()
+    show.columns = ['expt number', 'chunk_chars', 'overlap',
+                    'split', 'model', 'retriever', 'embedding', 'num_neighbors', 'Latency', 'Retrieval score',
+                    'Answer score']
+    show['expt number'] = show['expt number'].apply(lambda x: "Expt #: " + str(x + 1))
+    c = alt.Chart(show).mark_circle().encode(x='Retrieval score',
+                                             y='Answer score',
+                                             size=alt.Size('Latency'),
+                                             color='expt number',
+                                             tooltip=['expt number', 'Retrieval score', 'Latency', 'Answer score'])
+    st.altair_chart(c, use_container_width=True, theme="streamlit")
+
+else:
+
+    st.warning("Please input file and API key(s)!")

docs/karpathy-lex-pod/karpathy-pod-eval.csv

@@ -0,0 +1,3 @@
+"question","answer",
+"Why is the transformer architecture expressive in the forward pass?","The transformer architecture is expressive because it uses a general message passing scheme where nodes get to look at each other, decide what's interesting and then update each other.",
+"Why is next word prediction an effective training objective?", "On a sufficiently large dataset, the task of predicting the next word multi-tasks knowledge of a lot of things, including understanding of chemistry, physics, and human nature. You have to understand a lot about the world to make that prediction on an internet-scale dataset.",

docs/karpathy-lex-pod/karpathy-pod-eval.json

@@ -0,0 +1,12 @@
+[
+{"question": "Why is the transformer architecture expressive in the forward pass?",
+  "answer": "The transformer architecture is expressive because it uses a general message passing scheme where nodes get to look at each other, decide what's interesting and then update each other."},
+ {"question": "What design criteria does the Transformer meet?",
+  "answer": "The transformer is very expressive in a forward pass, optimizable in the backward pass using the techniques that we have such as gradient descent, and it can run efficiently on our hardware such as GPUs."},
+  {"question": "Why is next word prediction an effective training objective?",
+  "answer": "On a sufficiently large dataset, the task of predicting the next word multi-tasks knowledge of a lot of things, including understanding of chemistry, physics, and human nature. You have to understand a lot about the world to make that prediction on an internet-scale dataset."},
+  {"question": "What was the World Of Bits project and why did it fail?",
+  "answer": "World Of Bits was an effort to give AI access to tools, such as a keyboard and mouse, in order to complete tasks, such as complete bookings. It failed because it turned out that reinforcement learning is an extremely inefficient way of training neural networks. You take many actions, but you only get a sparse reward once in a while. Starting from scratch, it is very unlikely to stumble on the correct action - such as a booking - by chance at random, so the reward signal is very sparse."},
+ {"question": "Why can additional sensors be a liability in an autonomous vehicle system?",
+  "answer": "Each sensor adds complexity to the system. The hardware must be sourced, versioned, and maintain firmware. Software must ingest it, track versions. The cost of this additional bloat or entropy must be weighted against the added benefit of that particular sensor."}
+]

requirements.txt

@@ -0,0 +1,21 @@
+pandas==1.4.3
+fastapi==0.85.2
+langchain==0.0.141
+python-multipart==0.0.6
+uvicorn==0.18.3
+openai==0.27.0
+tiktoken==0.3.1
+faiss-cpu==1.7.3
+huggingface-hub==0.12.0
+anthropic==0.2.6
+llama-cpp-python==0.1.32
+pypdf==3.7.1
+filetype==1.2.0
+altair==4.2.2
+tokenizers==0.13.3
+sentence-transformers==2.2.2
+scikit-learn==1.2.1
+llama-index==0.4.35.post1
+streamlit==1.21.0
+gpt-index==0.5.16
+faiss-cpu==1.7.3

text_utils.py

@@ -0,0 +1,109 @@
+import re
+from langchain.prompts import PromptTemplate
+
+
+def clean_pdf_text(text: str) -> str:
+    """Cleans text extracted from a PDF file."""
+    # TODO: Remove References/Bibliography section.
+    return remove_citations(text)
+
+
+def remove_citations(text: str) -> str:
+    """Removes in-text citations from a string."""
+    # (Author, Year)
+    text = re.sub(r'\([A-Za-z0-9,.\s]+\s\d{4}\)', '', text)
+    # [1], [2], [3-5], [3, 33, 49, 51]
+    text = re.sub(r'\[[0-9,-]+(,\s[0-9,-]+)*\]', '', text)
+    return text
+
+
+template = """You are a teacher grading a quiz. 
+You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.
+Example Format:
+QUESTION: question here
+STUDENT ANSWER: student's answer here
+TRUE ANSWER: true answer here
+GRADE: CORRECT or INCORRECT here
+Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! 
+QUESTION: {query}
+STUDENT ANSWER: {result}
+TRUE ANSWER: {answer}
+GRADE:
+And explain why the STUDENT ANSWER is correct or incorrect.
+"""
+
+GRADE_ANSWER_PROMPT = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
+
+template = """You are a teacher grading a quiz. 
+You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.
+You are also asked to identify potential sources of bias in the question and in the true answer.
+Example Format:
+QUESTION: question here
+STUDENT ANSWER: student's answer here
+TRUE ANSWER: true answer here
+GRADE: CORRECT or INCORRECT here
+Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! 
+QUESTION: {query}
+STUDENT ANSWER: {result}
+TRUE ANSWER: {answer}
+GRADE:
+And explain why the STUDENT ANSWER is correct or incorrect, identify potential sources of bias in the QUESTION, and identify potential sources of bias in the TRUE ANSWER.
+"""
+
+GRADE_ANSWER_PROMPT_BIAS_CHECK = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
+
+template = """You are assessing a submitted student answer to a question relative to the true answer based on the provided criteria: 
+    
+    ***
+    QUESTION: {query}
+    ***
+    STUDENT ANSWER: {result}
+    ***
+    TRUE ANSWER: {answer}
+    ***
+    Criteria: 
+      relevance:  Is the submission referring to a real quote from the text?"
+      conciseness:  Is the answer concise and to the point?"
+      correct: Is the answer correct?"
+    ***
+    Does the submission meet the criterion? First, write out in a step by step manner your reasoning about the criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print the "CORRECT" or "INCORRECT" (without quotes or punctuation) on its own line corresponding to the correct answer.
+    Reasoning:
+"""
+
+GRADE_ANSWER_PROMPT_OPENAI = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
+
+template = """You are a teacher grading a quiz. 
+You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.
+Example Format:
+QUESTION: question here
+STUDENT ANSWER: student's answer here
+TRUE ANSWER: true answer here
+GRADE: CORRECT or INCORRECT here
+Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! 
+QUESTION: {query}
+STUDENT ANSWER: {result}
+TRUE ANSWER: {answer}
+GRADE:"""
+
+GRADE_ANSWER_PROMPT_FAST = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
+
+template = """ 
+    Given the question: \n
+    {query}
+    Decide if the following retrieved context is relevant: \n
+    {result}
+    Answer in the following format: \n
+    "Context is relevant: True or False." \n 
+    And explain why it supports or does not support the correct answer: {answer}"""
+
+GRADE_DOCS_PROMPT = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
+
+template = """ 
+    Given the question: \n
+    {query}
+    Decide if the following retrieved context is relevant to the {answer}: \n
+    {result}
+    Answer in the following format: \n
+    "Context is relevant: True or False." \n """
+
+GRADE_DOCS_PROMPT_FAST = PromptTemplate(input_variables=["query", "result", "answer"], template=template)