Update app.py

app.py

@@ -2,13 +2,10 @@ from gpt_index import GPTSimpleVectorIndex
 from langchain import OpenAI
 import gradio as gr
 from gradio import Interface, Textbox
-import sys
 import os
 import datetime
-import huggingface_hub
-from huggingface_hub import Repository
-from datetime import datetime
-import csv
+from datasets import load_dataset
+from huggingface_hub import HfFolder
 
 os.environ["OPENAI_API_KEY"] = os.environ['SECRET_CODE']
 
@@ -22,19 +19,15 @@ INDEX_FILE = os.path.join("data", INDEX_FILENAME)
 # we need a write access token.
 HF_TOKEN = os.environ.get("HF_TOKEN")
 print("HF TOKEN is none?", HF_TOKEN is None)
-print("HF hub ver", huggingface_hub.__version__)
 
-#Clones the distant repo to the local repo
-repo = Repository(
-    local_dir='data', 
-    clone_from=DATASET_REPO_URL, 
-    use_auth_token=HF_TOKEN)
-
-print(f"Repo local_dir: {repo.local_dir}")
-print(f"Repo files: {os.listdir(repo.local_dir)}")
+# Clones the distant repo to the local repo
+dataset = load_dataset(DATASET_REPO_URL)
+dataset_folder = HfFolder(dataset._data_files["train"][0].path).path
+print(f"Dataset folder: {dataset_folder}")
+print(f"Dataset files: {os.listdir(dataset_folder)}")
 
 def generate_text() -> str:
-    with open(DATA_FILE) as file:
+    with open(os.path.join(dataset_folder, DATA_FILENAME)) as file:
         text = ""
         for line in file:
             row_parts = line.strip().split(",")
@@ -45,30 +38,23 @@ def generate_text() -> str:
         return text if text else "No messages yet"
 
 
-def push_to_hub(commit_message):
-    repo.git_add(DATA_FILE)  # Add the updated data file to the staged changes
-    repo.git_commit(commit_message)  # Commit the changes
-    repo.push_to_hub(token=HF_TOKEN)  # Push the changes to the remote repository
-
-
 def store_message(chatinput: str, chatresponse: str):
     if chatinput and chatresponse:
-        with open(DATA_FILE, "a") as file:
-            file.write(f"{datetime.now()},{chatinput},{chatresponse}\n")
-            print(f"Wrote to datafile: {datetime.now()},{chatinput},{chatresponse}\n")
+        with open(os.path.join(dataset_folder, DATA_FILENAME), "a") as file:
+            file.write(f"{datetime.datetime.now()},{chatinput},{chatresponse}\n")
+            print(f"Wrote to datafile: {datetime.datetime.now()},{chatinput},{chatresponse}\n")
 
         # Push back to hub every N-th time the function is called
         if store_message.count_calls % 1 == 0:
             print("Pushing back to Hugging Face model hub")
-            # Call the push_to_hub() function to push the changes to the hub
-            push_to_hub(commit_message="Added new chat data") # Include commit_message parameter
+            dataset.commit("Added new chat data")  # Commit the changes
         store_message.count_calls += 1
         
         return generate_text()
 
 store_message.count_calls = 1  #initiases the count at one. We want to count how many messages stored before pushing back to repo.
 
-#gets the index file which is the context data
+# gets the index file which is the context data
 def get_index(index_file_path):
     if os.path.exists(index_file_path):
         index_size = os.path.getsize(index_file_path)