update file to save in dataset repo

app.py

@@ -3,14 +3,13 @@
 import os
 from utils.data_loader import download_dataset, save_metadata
 from utils.vector_utils import create_vector_db
-from config import v_auth_token, v_vector_folder, v_metadata_file, v_base_path
+from config import v_auth_token, v_vector_folder, v_metadata_file
 
 if __name__ == "__main__":
     if not v_auth_token:
         raise ValueError("Authentication token not found. Ensure 'hkey' is set as a secret in Hugging Face Spaces.")
 
     # Ensure writable base directory and subdirectories are initialized
-    os.makedirs(v_base_path, exist_ok=True)
     os.makedirs(v_vector_folder, exist_ok=True)
     if not os.path.exists(v_metadata_file):
         print("Metadata file not found. Creating a new one.")
@@ -22,4 +21,4 @@ if __name__ == "__main__":
     if not v_dataset_path:
         print("Vector database is up-to-date.")
     else:
-        create_vector_db(v_dataset_path, v_vector_folder)
+        create_vector_db(v_dataset_path, v_vector_folder, v_auth_token)

utils/data_loader.py

@@ -12,13 +12,16 @@ def download_dataset(v_auth_token, v_metadata_file, v_vector_folder):
         print("No updates detected. Skipping vector creation.")
         return False
 
-    # Download the dataset
+    # Define the specific file to download
+    v_filename = "train.csv"  # Replace this with the actual filename in your repository
     v_dataset_path = hf_hub_download(
         repo_id="vishalsh13/Dataset1",
         repo_type="dataset",
-        subfolder="data",
+        subfolder="data",  # Adjust or remove subfolder as needed
+        filename=v_filename,
         token=v_auth_token
     )
+
     print("Dataset downloaded successfully.")
     save_metadata(v_metadata_file, v_current_metadata)
     return v_dataset_path
@@ -28,22 +31,12 @@ def fetch_metadata():
     return {"dataset_version": "v1.0"}
 
 def load_metadata(v_metadata_file):
-    """
-    Load metadata from the file. If the file doesn't exist or is empty, return an empty dictionary.
-    """
     if os.path.exists(v_metadata_file):
         with open(v_metadata_file, "r") as file:
-            try:
-                return json.load(file)
-            except json.JSONDecodeError:
-                print(f"Metadata file {v_metadata_file} is empty or invalid. Initializing new metadata.")
-                return {}
+            return json.load(file)
     return {}
 
 def save_metadata(v_metadata_file, v_metadata):
-    """
-    Save metadata to the file. Creates the file if it doesn't exist.
-    """
     os.makedirs(os.path.dirname(v_metadata_file), exist_ok=True)
     with open(v_metadata_file, "w") as file:
-        json.dump(v_metadata, file, indent=4)
+        json.dump(v_metadata, file)

utils/vector_utils.py

@@ -1,11 +1,12 @@
 # utils/vector_utils.py
 
 import os
-import numpy as np
 import faiss
+import numpy as np
+from huggingface_hub import Repository
 from sentence_transformers import SentenceTransformer
 
-def create_vector_db(v_dataset_path, v_vector_folder):
+def create_vector_db(v_dataset_path, v_vector_folder, v_auth_token):
     # Initialize the model
     obj_model = SentenceTransformer('all-MiniLM-L6-v2')
 
@@ -15,10 +16,37 @@ def create_vector_db(v_dataset_path, v_vector_folder):
 
     v_embeddings = obj_model.encode(v_data)
 
-    # Save vectors
+    # Save vectors locally
     os.makedirs(v_vector_folder, exist_ok=True)
+    v_vector_file = os.path.join(v_vector_folder, "vector_index")
     v_index = faiss.IndexFlatL2(v_embeddings.shape[1])
     v_index.add(np.array(v_embeddings))
-    faiss.write_index(v_index, os.path.join(v_vector_folder, "vector_index"))
+    faiss.write_index(v_index, v_vector_file)
+
+    print(f"Vector database created and saved locally at {v_vector_file}")
+
+    # Save vector file back to Hugging Face dataset repository
+    upload_to_huggingface_repo(v_vector_file, v_auth_token)
+    print("Vector file successfully uploaded to Hugging Face dataset repository.")
+
+def upload_to_huggingface_repo(v_file_path, v_auth_token):
+    """
+    Upload the given file to the Hugging Face dataset repository.
+    """
+    v_repo_id = "vishalsh13/Dataset1"  # Replace with your repository name
+    v_repo = Repository(
+        local_dir="temp_repo",
+        clone_from=v_repo_id,
+        use_auth_token=v_auth_token
+    )
+
+    # Copy the file to the repository directory
+    os.makedirs(v_repo.local_dir, exist_ok=True)
+    v_dest_path = os.path.join(v_repo.local_dir, os.path.basename(v_file_path))
+    os.replace(v_file_path, v_dest_path)
 
-    print(f"Vector database created at {v_vector_folder}")
+    # Commit and push the changes
+    v_repo.git_add(v_dest_path)
+    v_repo.git_commit("Upload updated vector file.")
+    v_repo.git_push()
+    print(f"Uploaded {os.path.basename(v_file_path)} to Hugging Face repository: {v_repo_id}")