fixed app to remove dumb dependencies that arent used

app.deprocated

@@ -1,133 +0,0 @@
-'''
-
-CONFIG AND IMPORTS
-
-'''
-from config import default_config
-
-from types import SimpleNamespace
-import gradio as gr
-import os, random
-from pathlib import Path
-import tiktoken
-from getpass import getpass
-from rich.markdown import Markdown
-
-import openai
-import wandb
-from pprint import pprint
-from wandb.integration.openai import autolog
-from langchain.text_splitter import MarkdownHeaderTextSplitter
-
-
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.vectorstores import Chroma
-
-
-
-from tenacity import (
-    retry,
-    stop_after_attempt,
-    wait_random_exponential, # for exponential backoff
-)  
-
-
-
-
-if os.getenv("OPENAI_API_KEY") is None:
-  if any(['VSCODE' in x for x in os.environ.keys()]):
-    print('Please enter password in the VS Code prompt at the top of your VS Code window!')
-  os.environ["OPENAI_API_KEY"] = getpass("Paste your OpenAI key from: https://platform.openai.com/account/api-keys\n")
-  openai.api_key = os.getenv("OPENAI_API_KEY", "")
-
-assert os.getenv("OPENAI_API_KEY", "").startswith("sk-"), "This doesn't look like a valid OpenAI API key"
-print("OpenAI API key configured")
-
-
-
-
-
-
-def find_nearest_neighbor(argument=""):
-    '''
-        INPUT:
-            argument (str)
-            vectorDB??
-        RETURN the nearest neighbor in vectorDB to argument
-    '''
-
-    md = ""
-    print(argument)
-    directory_path = "../../safety_docs"
-
-    for filename in os.listdir(directory_path):
-        if filename.endswith(".md"):
-            with open(os.path.join(directory_path, filename), 'r') as file:
-                content = file.read()
-                md = md + content
-
-    markdown_document = md
-
-    headers_to_split_on = [
-        ("#", "Header 1"),
-        ("##", "Header 2"),
-        ("###", "Header 3"),
-    ]
-
-    markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
-    md_header_splits = markdown_splitter.split_text(markdown_document)
-
-    embeddings = OpenAIEmbeddings()
-    db = Chroma.from_documents(md_header_splits, embeddings)
-
-    retriever = db.as_retriever(search_kwargs=dict(k=1))
-
-    docs = retriever.get_relevant_documents(argument)
-
-    return docs[0].metadata["Header 1"]
-    
-
-
-def get_gpt_response(argument, user_prompt, system_prompt=default_config.system_prompt, model=default_config.model_name, n=1, max_tokens=200):
-    '''
-    INPUT: 
-    Argument
-    user_prompt
-    system_prompt
-    model
-    '''
-
-    @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(2))
-    def completion_with_backoff(**kwargs):
-        return openai.ChatCompletion.create(**kwargs)
-    
-    messages=[
-            {"role": "system", "content": system_prompt},
-            {"role": "user", "content": user_prompt},
-        ]
-    responses = completion_with_backoff(
-        model=model,
-        messages=messages,
-        n = n,
-        max_tokens=max_tokens
-        )
-    for response in responses.choices:
-        generation = response.message.content
-        return generation
-    
-
-def greet(argument):
-    nearest_neighbor = find_nearest_neighbor(argument)
-    user_prompt = default_config.user_prompt_1 + argument + default_config.user_prompt_2
-    response = get_gpt_response(argument, user_prompt)
-    return "Hello " + argument + "\n nice argument, it actually is a common one: " + nearest_neighbor + "\n gpt response: \n" + response
-    
-
-demo = gr.Interface(
-    fn=greet,
-    inputs=gr.Textbox(lines=2, placeholder="poob here"),
-    outputs="text"
-)
-
-demo.queue(max_size=20)
-demo.launch()

app.py

@@ -11,27 +11,16 @@ import os, random
 from pathlib import Path
 import tiktoken
 from getpass import getpass
-from rich.markdown import Markdown
 
 import openai
-import wandb
-from pprint import pprint
-from wandb.integration.openai import autolog
 from langchain.text_splitter import MarkdownHeaderTextSplitter
 import numpy as np
 
 from langchain.embeddings import OpenAIEmbeddings
-from langchain.vectorstores import Chroma
+# from langchain.vectorstores import Chroma
 
 
 
-from tenacity import (
-    retry,
-    stop_after_attempt,
-    wait_random_exponential, # for exponential backoff
-)  
-
-
 
 
 if os.getenv("OPENAI_API_KEY") is None:

app2.py.deprocated

@@ -1,195 +0,0 @@
-'''
-
-CONFIG AND IMPORTS
-
-'''
-from config import default_config
-
-from types import SimpleNamespace
-import gradio as gr
-import os, random
-from pathlib import Path
-import tiktoken
-from getpass import getpass
-from rich.markdown import Markdown
-
-import openai
-import wandb
-from pprint import pprint
-from wandb.integration.openai import autolog
-from langchain.text_splitter import MarkdownHeaderTextSplitter
-
-
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.vectorstores import Chroma
-
-
-
-from tenacity import (
-    retry,
-    stop_after_attempt,
-    wait_random_exponential, # for exponential backoff
-)  
-
-
-
-
-if os.getenv("OPENAI_API_KEY") is None:
-  if any(['VSCODE' in x for x in os.environ.keys()]):
-    print('Please enter password in the VS Code prompt at the top of your VS Code window!')
-  os.environ["OPENAI_API_KEY"] = getpass("Paste your OpenAI key from: https://platform.openai.com/account/api-keys\n")
-  openai.api_key = os.getenv("OPENAI_API_KEY", "")
-
-assert os.getenv("OPENAI_API_KEY", "").startswith("sk-"), "This doesn't look like a valid OpenAI API key"
-print("OpenAI API key configured")
-
-
-
-
-
-
-def find_nearest_neighbor(argument="", max_args_in_output=3):
-    '''
-        INPUT:
-            argument (string)
-
-        RETURN the nearest neighbor(s) in vectorDB to argument as string
-    '''
-
-    md = ""
-    print(argument)
-    directory_path = "../../safety_docs"
-
-    for filename in os.listdir(directory_path):
-        if filename.endswith(".md"):
-            with open(os.path.join(directory_path, filename), 'r') as file:
-                content = file.read()
-                md = md + content
-
-    markdown_document = md
-
-    headers_to_split_on = [
-        ("#", "Header 1"),
-        ("##", "Header 2"),
-        ("###", "Header 3"),
-    ]
-
-    markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
-    md_header_splits = markdown_splitter.split_text(markdown_document)
-
-    embeddings = OpenAIEmbeddings()
-    db = Chroma.from_documents(md_header_splits, embeddings)
-
-    retriever = db.as_retriever(search_kwargs=dict(k=11))
-
-    docs = retriever.get_relevant_documents(argument)
-
-    output = "" # output to return, a list of common args
-    seen = set() # which documents have been added to output
-    count = 0 # count how many embeddings have been added to output
-    for doc in docs:
-        if doc.metadata["Header 1"] not in seen: 
-            output = output + doc.metadata["Header 1"] + '\n'
-            count = count + 1
-        seen.add(doc.metadata["Header 1"])
-        if count >= max_args_in_output:
-            break
-        
-    return output
-    
-
-
-def get_gpt_response(argument, user_prompt, system_prompt=default_config.system_prompt, model=default_config.model_name, n=1, max_tokens=200):
-    '''
-    INPUT: 
-    Argument
-    user_prompt
-    system_prompt
-    model
-    '''
-
-    @retry(wait=wait_random_exponential(min=1, max=3), stop=stop_after_attempt(1))
-    def completion_with_backoff(**kwargs):
-        return openai.ChatCompletion.create(**kwargs)
-    
-    messages=[
-            {"role": "system", "content": system_prompt},
-            {"role": "user", "content": user_prompt},
-        ]
-    responses = completion_with_backoff(
-        model=model,
-        messages=messages,
-        n = n,
-        max_tokens=max_tokens
-        )
-    for response in responses.choices:
-        generation = response.message.content
-        return generation
-    
-
-def greet(argument):
-    nearest_neighbor = find_nearest_neighbor(argument)
-    user_prompt = default_config.user_prompt_1 + argument + default_config.user_prompt_2
-    # response = get_gpt_response(argument, user_prompt)
-    response = "chatbot response here"
-    return "Hello " + "\n We think your argument matches common arguments in our database, is it one of these?:\n " + nearest_neighbor + "\n\n\n ------------------------- \n\n\n Lengthy response: \n" + response
-    
-
-demo = gr.Interface(
-    fn=greet,
-    inputs=gr.Textbox(lines=2, placeholder="Anything past 200 tokens (roughly 200 words) will be cutoff. Please enter <=1 paragraph"),
-    outputs="text"
-)
-
-# demo.queue(max_size=20)
-demo.launch()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-def find_nearest_neighbor(argument=""):
-    '''
-        INPUT:
-            argument (string)
-            
-        RETURN the nearest neighbor(s) in vectorDB to argument as string
-    '''
-
-    md = ""
-    directory_path = "../../safety_docs"
-
-    for filename in os.listdir(directory_path):
-        if filename.endswith(".md"):
-            with open(os.path.join(directory_path, filename), 'r') as file:
-                content = file.read()
-                md = md + content
-
-    markdown_document = md
-    headers_to_split_on = [
-        ("#", "Header 1"),
-        ("##", "Header 2"),
-        ("###", "Header 3"),
-    ]
-
-    markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
-    md_header_splits = markdown_splitter.split_text(markdown_document)
-
-    embeddings = OpenAIEmbeddings()
-    db = Chroma.from_documents(md_header_splits, embeddings)
-
-    retriever = db.as_retriever(search_kwargs=dict(k=11))
-
-    docs = retriever.get_relevant_documents(argument)
-
-    # return the content of the nearest neighbor document
-    return docs[0].metadata["Header 1"]

ingest.py

@@ -1,208 +0,0 @@
-"""Ingest a directory of documentation files into a vector store and store the relevant artifacts in Weights & Biases"""
-import argparse
-import json
-import logging
-import os
-import pathlib
-from typing import List, Tuple
-
-import langchain
-import wandb
-from langchain.cache import SQLiteCache
-from langchain.docstore.document import Document
-from langchain.document_loaders import UnstructuredMarkdownLoader
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.text_splitter import MarkdownTextSplitter
-from langchain.vectorstores import Chroma
-
-langchain.llm_cache = SQLiteCache(database_path="langchain.db")
-
-logger = logging.getLogger(__name__)
-
-
-def load_documents(data_dir: str) -> List[Document]:
-    """Load documents from a directory of markdown files
-
-    Args:
-        data_dir (str): The directory containing the markdown files
-
-    Returns:
-        List[Document]: A list of documents
-    """
-    md_files = list(map(str, pathlib.Path(data_dir).glob("*.md")))
-    documents = [
-        UnstructuredMarkdownLoader(file_path=file_path).load()[0]
-        for file_path in md_files
-    ]
-    return documents
-
-
-def chunk_documents(
-    documents: List[Document], chunk_size: int = 500, chunk_overlap=0
-) -> List[Document]:
-    """Split documents into chunks
-
-    Args:
-        documents (List[Document]): A list of documents to split into chunks
-        chunk_size (int, optional): The size of each chunk. Defaults to 500.
-        chunk_overlap (int, optional): The number of tokens to overlap between chunks. Defaults to 0.
-
-    Returns:
-        List[Document]: A list of chunked documents.
-    """
-    markdown_text_splitter = MarkdownTextSplitter(
-        chunk_size=chunk_size, chunk_overlap=chunk_overlap
-    )
-    split_documents = markdown_text_splitter.split_documents(documents)
-    return split_documents
-
-
-def create_vector_store(
-    documents,
-    vector_store_path: str = "./vector_store",
-) -> Chroma:
-    """Create a ChromaDB vector store from a list of documents
-
-    Args:
-        documents (_type_): A list of documents to add to the vector store
-        vector_store_path (str, optional): The path to the vector store. Defaults to "./vector_store".
-
-    Returns:
-        Chroma: A ChromaDB vector store containing the documents.
-    """
-    api_key = os.environ.get("OPENAI_API_KEY", None)
-    embedding_function = OpenAIEmbeddings(openai_api_key=api_key)
-    vector_store = Chroma.from_documents(
-        documents=documents,
-        embedding=embedding_function,
-        persist_directory=vector_store_path,
-    )
-    vector_store.persist()
-    return vector_store
-
-
-def log_dataset(documents: List[Document], run: "wandb.run"):
-    """Log a dataset to wandb
-
-    Args:
-        documents (List[Document]): A list of documents to log to a wandb artifact
-        run (wandb.run): The wandb run to log the artifact to.
-    """
-    document_artifact = wandb.Artifact(name="documentation_dataset", type="dataset")
-    with document_artifact.new_file("documents.json") as f:
-        for document in documents:
-            f.write(document.json() + "\n")
-
-    run.log_artifact(document_artifact)
-
-
-def log_index(vector_store_dir: str, run: "wandb.run"):
-    """Log a vector store to wandb
-
-    Args:
-        vector_store_dir (str): The directory containing the vector store to log
-        run (wandb.run): The wandb run to log the artifact to.
-    """
-    index_artifact = wandb.Artifact(name="vector_store", type="search_index")
-    index_artifact.add_dir(vector_store_dir)
-    run.log_artifact(index_artifact)
-
-
-def log_prompt(prompt: dict, run: "wandb.run"):
-    """Log a prompt to wandb
-
-    Args:
-        prompt (str): The prompt to log
-        run (wandb.run): The wandb run to log the artifact to.
-    """
-    prompt_artifact = wandb.Artifact(name="chat_prompt", type="prompt")
-    with prompt_artifact.new_file("prompt.json") as f:
-        f.write(json.dumps(prompt))
-    run.log_artifact(prompt_artifact)
-
-
-def ingest_data(
-    docs_dir: str,
-    chunk_size: int,
-    chunk_overlap: int,
-    vector_store_path: str,
-) -> Tuple[List[Document], Chroma]:
-    """Ingest a directory of markdown files into a vector store
-
-    Args:
-        docs_dir (str):
-        chunk_size (int):
-        chunk_overlap (int):
-        vector_store_path (str):
-
-
-    """
-    # load the documents
-    documents = load_documents(docs_dir)
-    # split the documents into chunks
-    split_documents = chunk_documents(documents, chunk_size, chunk_overlap)
-    # create document embeddings and store them in a vector store
-    vector_store = create_vector_store(split_documents, vector_store_path)
-    return split_documents, vector_store
-
-
-def get_parser():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--docs_dir",
-        type=str,
-        required=True,
-        help="The directory containing the wandb documentation",
-    )
-    parser.add_argument(
-        "--chunk_size",
-        type=int,
-        default=500,
-        help="The number of tokens to include in each document chunk",
-    )
-    parser.add_argument(
-        "--chunk_overlap",
-        type=int,
-        default=0,
-        help="The number of tokens to overlap between document chunks",
-    )
-    parser.add_argument(
-        "--vector_store",
-        type=str,
-        default="./vector_store",
-        help="The directory to save or load the Chroma db to/from",
-    )
-    parser.add_argument(
-        "--prompt_file",
-        type=pathlib.Path,
-        default="./chat_prompt.json",
-        help="The path to the chat prompt to use",
-    )
-    parser.add_argument(
-        "--wandb_project",
-        default="llmapps",
-        type=str,
-        help="The wandb project to use for storing artifacts",
-    )
-
-    return parser
-
-
-def main():
-    parser = get_parser()
-    args = parser.parse_args()
-    run = wandb.init(project=args.wandb_project, config=args)
-    documents, vector_store = ingest_data(
-        docs_dir=args.docs_dir,
-        chunk_size=args.chunk_size,
-        chunk_overlap=args.chunk_overlap,
-        vector_store_path=args.vector_store,
-    )
-    log_dataset(documents, run)
-    log_index(args.vector_store, run)
-    log_prompt(json.load(args.prompt_file.open("r")), run)
-    run.finish()
-
-
-if __name__ == "__main__":
-    main()