Initial Draft

Dashboard.py

@@ -6,18 +6,39 @@ import os
 ################################
 ######### Variables ############
 ################################
-
 HF_API_KEY = os.environ.get("HF_API_KEY")
 DATA_SET = os.environ.get("DATA_SET")
 
-st.write(HF_API_KEY)
-st.write(DATA_SET)
-
 ################################
 ####### GenericFunctions #######
 ################################
+def load_dataset():
+    dataset = load_dataset(DATA_SET, use_auth_token=True)
+    
+def save_dataset():
+    # Load a dataset from the Hugging Face Hub
+    load_dataset()
+
+    # Create a directory to save the files
+    output_directory = "data"
+    os.makedirs(output_directory, exist_ok=True)
 
+    # Iterate through the dataset and save each file to the data folder
+    for split in dataset.keys():
+        for i, example in enumerate(dataset[split]):
+            file_content = example['column_name']  # Replace 'column_name' with the actual column containing file data
+            file_name = f"{split}_{i}.txt"  # Create a unique file name
+            file_path = os.path.join(output_directory, file_name)
+    
+            with open(file_path, 'w', encoding='utf-8') as file:
+                file.write(file_content)
+    
+            print(f"Saved: {file_path}")
 
+    print("All files saved successfully.")
+
+    
 ################################
 ####### Display of data ########
-################################
+################################
+save_dataset()