Refactor training data export function and add dataset structure repair tool

- Updated the export_training_data_action function call to change the order of parameters in app.py.
- Added new constants for training data paths in settings.py.
- Implemented ensure_dataset_directory function in dataset.py to create missing directories for training data.
- Enhanced dataset structure repair tool in repair_dataset_structure.py to move misplaced chat files and fix duplicated paths.
- Added logging functionality to track the repair process and errors.

app.py

@@ -1375,7 +1375,7 @@ with gr.Blocks(css="""
             
             # Export training data
             export_btn.click(
-                fn=lambda min_r, path: export_training_data_action(min_r, path, chat_evaluator),
+                fn=lambda min_r, path: export_training_data_action(chat_evaluator, min_r, path),
                 inputs=[min_rating, export_path],
                 outputs=[export_status]
             )

config/settings.py

@@ -29,6 +29,9 @@ DATASET_FINE_TUNED_PATH = "fine_tuned_models"
 DATASET_ANNOTATIONS_PATH = "annotations"
 DATASET_ERROR_LOGS_PATH = "error_logs"
 DATASET_PREFERENCES_PATH = "preferences/user_preferences.json"
+# Adding training data paths
+DATASET_TRAINING_DATA_PATH = "training_data"
+DATASET_TRAINING_LOGS_PATH = "training_logs"
 
 # Temporary storage (using system temp directory)
 TEMP_DIR = tempfile.gettempdir()

dataset.py

@@ -5,18 +5,43 @@ from config.settings import (
     DATASET_CHAT_HISTORY_PATH,
     DATASET_FINE_TUNED_PATH,
     DATASET_ANNOTATIONS_PATH,
+    DATASET_TRAINING_DATA_PATH,
+    DATASET_TRAINING_LOGS_PATH,
     HF_TOKEN
 )
 
 api = HfApi(token=HF_TOKEN)
 dataset_name = DATASET_ID
 
+def ensure_dataset_directory(directory: str) -> None:
+    """
+    Check if directory exists in dataset and create if not
+    """
+    try:
+        files = api.list_repo_files(repo_id=dataset_name, repo_type="dataset")
+        if not any(f.startswith(f"{directory}/") for f in files):
+            api.upload_file(
+                path_or_fileobj=b"",
+                path_in_repo=f"{directory}/.gitkeep",
+                repo_id=dataset_name,
+                repo_type="dataset"
+            )
+            print(f"✓ Created missing directory: {directory}")
+    except Exception as e:
+        print(f"Error ensuring directory {directory}: {str(e)}")
+
+# Ensure training directories exist
+ensure_dataset_directory(DATASET_TRAINING_DATA_PATH)
+ensure_dataset_directory(DATASET_TRAINING_LOGS_PATH)
+
 # Initialize dataset structure
 directories = [
     DATASET_VECTOR_STORE_PATH,
     DATASET_CHAT_HISTORY_PATH,
     DATASET_FINE_TUNED_PATH,
-    DATASET_ANNOTATIONS_PATH
+    DATASET_ANNOTATIONS_PATH,
+    DATASET_TRAINING_DATA_PATH,
+    DATASET_TRAINING_LOGS_PATH
 ]
 
 try:

repair_dataset_structure.log

@@ -0,0 +1,178 @@
+2025-04-10 17:42:04,332 - __main__ - INFO - === Starting Dataset Structure Repair ===
+2025-04-10 17:42:04,333 - __main__ - INFO - Dataset ID: Rulga/status-law-knowledge-base
+2025-04-10 17:42:04,727 - __main__ - INFO - Found 3 misplaced chat files
+2025-04-10 17:42:08,843 - __main__ - INFO - Moved annotations/annotation_01be5785-10a7-460b-9374-f749897987c7.json to chat_history/annotations/annotation_01be5785-10a7-460b-9374-f749897987c7.json
+2025-04-10 17:42:12,475 - __main__ - INFO - Moved annotations/annotation_46035aa7-b1c9-4c33-af58-0cf335f6727c.json to chat_history/annotations/annotation_46035aa7-b1c9-4c33-af58-0cf335f6727c.json
+2025-04-10 17:42:16,223 - __main__ - INFO - Moved annotations/annotation_6f98aeb4-58ca-48a4-a236-2719592c67b2.json to chat_history/annotations/annotation_6f98aeb4-58ca-48a4-a236-2719592c67b2.json
+2025-04-10 17:42:16,223 - __main__ - INFO - Successfully moved 3 files
+2025-04-10 17:42:16,223 - __main__ - INFO - === Repair Complete ===
+2025-04-10 18:43:48,372 - __main__ - INFO - === Starting Dataset Structure Repair ===
+2025-04-10 18:43:48,372 - __main__ - INFO - Dataset ID: Rulga/status-law-knowledge-base
+2025-04-10 18:43:48,893 - __main__ - INFO - Found 42 misplaced chat files
+2025-04-10 18:43:53,235 - __main__ - INFO - Moved chat_history\20250403-003446.json to chat_history/chat_history\20250403-003446.json
+2025-04-10 18:43:57,233 - __main__ - INFO - Moved chat_history\20250403-004332.json to chat_history/chat_history\20250403-004332.json
+2025-04-10 18:44:01,825 - __main__ - INFO - Moved chat_history\20250403-005218.json to chat_history/chat_history\20250403-005218.json
+2025-04-10 18:44:06,138 - __main__ - INFO - Moved chat_history\20250403-005243.json to chat_history/chat_history\20250403-005243.json
+2025-04-10 18:44:10,245 - __main__ - INFO - Moved chat_history\20250403-005335.json to chat_history/chat_history\20250403-005335.json
+2025-04-10 18:44:14,219 - __main__ - INFO - Moved chat_history\20250403-115212.json to chat_history/chat_history\20250403-115212.json
+2025-04-10 18:44:19,303 - __main__ - INFO - Moved chat_history\20250403-154557.json to chat_history/chat_history\20250403-154557.json
+2025-04-10 18:44:22,916 - __main__ - INFO - Moved chat_history\20250403-170727.json to chat_history/chat_history\20250403-170727.json
+2025-04-10 18:44:26,650 - __main__ - INFO - Moved chat_history\20250403-171345.json to chat_history/chat_history\20250403-171345.json
+2025-04-10 18:44:31,024 - __main__ - INFO - Moved chat_history\20250403-180146.json to chat_history/chat_history\20250403-180146.json
+2025-04-10 18:44:34,740 - __main__ - INFO - Moved chat_history\20250403-181618.json to chat_history/chat_history\20250403-181618.json
+2025-04-10 18:44:38,314 - __main__ - INFO - Moved chat_history\20250403-182229.json to chat_history/chat_history\20250403-182229.json
+2025-04-10 18:44:41,931 - __main__ - INFO - Moved chat_history\20250404-131413.json to chat_history/chat_history\20250404-131413.json
+2025-04-10 18:44:45,580 - __main__ - INFO - Moved chat_history\20250404-131503.json to chat_history/chat_history\20250404-131503.json
+2025-04-10 18:44:49,412 - __main__ - INFO - Moved chat_history\20250404-134635.json to chat_history/chat_history\20250404-134635.json
+2025-04-10 18:44:53,247 - __main__ - INFO - Moved chat_history\20250404-140648.json to chat_history/chat_history\20250404-140648.json
+2025-04-10 18:44:56,894 - __main__ - INFO - Moved chat_history\20250404-140914.json to chat_history/chat_history\20250404-140914.json
+2025-04-10 18:45:00,593 - __main__ - INFO - Moved chat_history\20250404-140955.json to chat_history/chat_history\20250404-140955.json
+2025-04-10 18:45:04,591 - __main__ - INFO - Moved chat_history\20250404-175337.json to chat_history/chat_history\20250404-175337.json
+2025-04-10 18:45:08,298 - __main__ - INFO - Moved chat_history\20250404-180019.json to chat_history/chat_history\20250404-180019.json
+2025-04-10 18:45:11,886 - __main__ - INFO - Moved chat_history\20250404-180053.json to chat_history/chat_history\20250404-180053.json
+2025-04-10 18:45:15,675 - __main__ - INFO - Moved chat_history\20250404-180123.json to chat_history/chat_history\20250404-180123.json
+2025-04-10 18:45:19,630 - __main__ - INFO - Moved chat_history\20250404-180651.json to chat_history/chat_history\20250404-180651.json
+2025-04-10 18:45:23,351 - __main__ - INFO - Moved chat_history\20250404-180725.json to chat_history/chat_history\20250404-180725.json
+2025-04-10 18:45:27,772 - __main__ - INFO - Moved chat_history\20250404-180940.json to chat_history/chat_history\20250404-180940.json
+2025-04-10 18:45:31,678 - __main__ - INFO - Moved chat_history\20250404-181628.json to chat_history/chat_history\20250404-181628.json
+2025-04-10 18:45:35,959 - __main__ - INFO - Moved chat_history\20250404-181730.json to chat_history/chat_history\20250404-181730.json
+2025-04-10 18:45:39,765 - __main__ - INFO - Moved chat_history\20250404-184103.json to chat_history/chat_history\20250404-184103.json
+2025-04-10 18:45:43,638 - __main__ - INFO - Moved chat_history\20250404-184613.json to chat_history/chat_history\20250404-184613.json
+2025-04-10 18:45:47,378 - __main__ - INFO - Moved chat_history\20250404-184705.json to chat_history/chat_history\20250404-184705.json
+2025-04-10 18:45:51,172 - __main__ - INFO - Moved chat_history\20250404-185243.json to chat_history/chat_history\20250404-185243.json
+2025-04-10 18:45:54,867 - __main__ - INFO - Moved chat_history\20250404-185810.json to chat_history/chat_history\20250404-185810.json
+2025-04-10 18:45:58,349 - __main__ - INFO - Moved chat_history\20250404-202051.json to chat_history/chat_history\20250404-202051.json
+2025-04-10 18:46:03,364 - __main__ - INFO - Moved chat_history\20250407-174318.json to chat_history/chat_history\20250407-174318.json
+2025-04-10 18:46:06,933 - __main__ - INFO - Moved chat_history\20250407-175851.json to chat_history/chat_history\20250407-175851.json
+2025-04-10 18:46:10,714 - __main__ - INFO - Moved chat_history\20250408-135920.json to chat_history/chat_history\20250408-135920.json
+2025-04-10 18:46:14,230 - __main__ - INFO - Moved chat_history\20250408-143211.json to chat_history/chat_history\20250408-143211.json
+2025-04-10 18:46:17,815 - __main__ - INFO - Moved chat_history\20250408-152804.json to chat_history/chat_history\20250408-152804.json
+2025-04-10 18:46:21,415 - __main__ - INFO - Moved chat_history\20250408-161228.json to chat_history/chat_history\20250408-161228.json
+2025-04-10 18:46:25,573 - __main__ - INFO - Moved chat_history\20250408-165140.json to chat_history/chat_history\20250408-165140.json
+2025-04-10 18:46:29,149 - __main__ - INFO - Moved chat_history\20250408-165215.json to chat_history/chat_history\20250408-165215.json
+2025-04-10 18:46:32,798 - __main__ - INFO - Moved chat_history\20250408-165308.json to chat_history/chat_history\20250408-165308.json
+2025-04-10 18:46:32,798 - __main__ - INFO - Successfully moved 42 files from root to chat_history
+2025-04-10 18:46:32,814 - __main__ - INFO - === Repair Complete ===
+2025-04-10 18:52:09,005 - __main__ - INFO - === Starting Dataset Structure Repair ===
+2025-04-10 18:52:09,005 - __main__ - INFO - Dataset ID: Rulga/status-law-knowledge-base
+2025-04-10 18:52:09,005 - __main__ - INFO - === Starting Path Fix ===
+2025-04-10 18:52:09,415 - __main__ - INFO - Found 42 files with duplicated chat_history path
+2025-04-10 18:52:17,036 - __main__ - INFO - Renamed chat_history/chat_history\20250403-003446.json to chat_history/20250403-003446.json
+2025-04-10 18:52:21,492 - __main__ - INFO - Renamed chat_history/chat_history\20250403-004332.json to chat_history/20250403-004332.json
+2025-04-10 18:52:25,225 - __main__ - INFO - Renamed chat_history/chat_history\20250403-005218.json to chat_history/20250403-005218.json
+2025-04-10 18:52:29,282 - __main__ - INFO - Renamed chat_history/chat_history\20250403-005243.json to chat_history/20250403-005243.json
+2025-04-10 18:52:32,882 - __main__ - INFO - Renamed chat_history/chat_history\20250403-005335.json to chat_history/20250403-005335.json
+2025-04-10 18:52:36,438 - __main__ - INFO - Renamed chat_history/chat_history\20250403-115212.json to chat_history/20250403-115212.json
+2025-04-10 18:52:40,280 - __main__ - INFO - Renamed chat_history/chat_history\20250403-154557.json to chat_history/20250403-154557.json
+2025-04-10 18:52:44,213 - __main__ - INFO - Renamed chat_history/chat_history\20250403-170727.json to chat_history/20250403-170727.json
+2025-04-10 18:52:48,115 - __main__ - INFO - Renamed chat_history/chat_history\20250403-171345.json to chat_history/20250403-171345.json
+2025-04-10 18:52:51,976 - __main__ - INFO - Renamed chat_history/chat_history\20250403-180146.json to chat_history/20250403-180146.json
+2025-04-10 18:52:55,930 - __main__ - INFO - Renamed chat_history/chat_history\20250403-181618.json to chat_history/20250403-181618.json
+2025-04-10 18:52:59,738 - __main__ - INFO - Renamed chat_history/chat_history\20250403-182229.json to chat_history/20250403-182229.json
+2025-04-10 18:53:05,572 - __main__ - INFO - Renamed chat_history/chat_history\20250404-131413.json to chat_history/20250404-131413.json
+2025-04-10 18:53:09,406 - __main__ - INFO - Renamed chat_history/chat_history\20250404-131503.json to chat_history/20250404-131503.json
+2025-04-10 18:53:13,212 - __main__ - INFO - Renamed chat_history/chat_history\20250404-134635.json to chat_history/20250404-134635.json
+2025-04-10 18:53:17,506 - __main__ - INFO - Renamed chat_history/chat_history\20250404-140648.json to chat_history/20250404-140648.json
+2025-04-10 18:53:21,872 - __main__ - INFO - Renamed chat_history/chat_history\20250404-140914.json to chat_history/20250404-140914.json
+2025-04-10 18:53:25,516 - __main__ - INFO - Renamed chat_history/chat_history\20250404-140955.json to chat_history/20250404-140955.json
+2025-04-10 18:53:29,177 - __main__ - INFO - Renamed chat_history/chat_history\20250404-175337.json to chat_history/20250404-175337.json
+2025-04-10 18:53:32,998 - __main__ - INFO - Renamed chat_history/chat_history\20250404-180019.json to chat_history/20250404-180019.json
+2025-04-10 18:53:36,716 - __main__ - INFO - Renamed chat_history/chat_history\20250404-180053.json to chat_history/20250404-180053.json
+2025-04-10 18:53:40,483 - __main__ - INFO - Renamed chat_history/chat_history\20250404-180123.json to chat_history/20250404-180123.json
+2025-04-10 18:53:43,269 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-180651.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f791-503acbd66e4aeb5f2ba7abd9;11c9bb9a-ac35-4c84-ba83-0a1f415ee40c)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:53:46,037 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-180725.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f794-490d14f53aebbbb72066ab6f;131385d7-3a06-4037-a1d0-0a8ba167c80d)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:53:49,886 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-180940.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f798-2e222f1d59ac3dc41de64dfb;aecbf90a-b7b0-43ca-b4b1-fffd4d6ab70d)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:53:52,612 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-181628.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f79b-09915158005dfa7c5a56a15d;3d5cecd6-e1a3-44f1-8234-8ab0a873b718)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:53:55,411 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-181730.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f79d-64e62bbb44ac2ef51c8518b0;927958ad-8808-4eb2-96b3-0528085c7396)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:53:58,337 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-184103.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7a0-5d8a1a2c1070cf2648e0ec44;37fccdca-bcc5-40d7-85f6-f26caa2489e6)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:01,048 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-184613.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7a3-448e4bda194517cf7a8285ed;14c70e24-3545-4b6a-841b-2bbac1e9bc6b)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:03,937 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-184705.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7a6-7655014d5e3a972d08815ddc;3cd55990-22c6-499a-95f1-f662d12c51b8)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:06,653 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-185243.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7a9-2eced6ec760924e43262136d;633dd326-9951-49b0-9b52-57825ae88ed0)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:09,403 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-185810.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7ab-0d9b2d483b6902386b048933;b95da74a-3609-454d-a0c0-0a7f14b6d6de)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:12,136 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-202051.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7ae-52d0acc90186a3dc4b9254a6;6c44b4d6-6627-4b01-8a50-a4fa21959fdb)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:15,081 - __main__ - ERROR - Error processing file chat_history/chat_history\20250407-174318.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7b1-4e99483725d815ad61bbaacc;6fb9057f-8d66-43ac-b603-789f4f7ba815)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:17,822 - __main__ - ERROR - Error processing file chat_history/chat_history\20250407-175851.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7b4-2b9c83346c295d82770bd968;8e704238-9252-426b-b350-e3f52fd2cbb0)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:20,537 - __main__ - ERROR - Error processing file chat_history/chat_history\20250408-135920.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7b6-6ea689051ea0cca350f3ca17;0fa53b91-119c-47b2-9761-93ea7d93a993)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:23,255 - __main__ - ERROR - Error processing file chat_history/chat_history\20250408-143211.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7b9-25cbff3e1f06fa441785271c;6c6a1190-6791-4167-9f75-a668903be2e1)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:27,172 - __main__ - ERROR - Error processing file chat_history/chat_history\20250408-152804.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7bd-5f8da572682024ef3aeecadd;65480ebf-2375-4cd8-95f3-bc841d8dc171)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:29,899 - __main__ - ERROR - Error processing file chat_history/chat_history\20250408-161228.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7c0-75a715fb768bdb2a68ccf46f;95d56ef4-8de8-40c9-9297-e7a8281a13d6)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:32,895 - __main__ - ERROR - Error processing file chat_history/chat_history\20250408-165140.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7c3-5f70e9ba565434fc7a3c5e18;275dda97-05cc-4a23-9baf-a8f43ede9d62)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:35,623 - __main__ - ERROR - Error processing file chat_history/chat_history\20250408-165215.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7c6-133de4eb4630a67f0b375fc6;046200b9-aae9-42e9-85ed-e3a684b164d4)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:38,329 - __main__ - ERROR - Error processing file chat_history/chat_history\20250408-165308.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f7c8-7723083e298904f14a1af942;fc2bd99a-4fcf-4936-a15c-acbb481941e7)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:54:38,329 - __main__ - INFO - Successfully renamed 22 files
+2025-04-10 18:54:38,329 - __main__ - WARNING - Failed to process 20 files
+2025-04-10 18:54:38,329 - __main__ - INFO - === Repair Complete ===
+2025-04-10 18:59:28,315 - __main__ - INFO - === Starting Dataset Structure Repair ===
+2025-04-10 18:59:28,318 - __main__ - INFO - Dataset ID: Rulga/status-law-knowledge-base
+2025-04-10 18:59:28,318 - __main__ - INFO - === Starting Path Fix ===
+2025-04-10 18:59:28,704 - __main__ - INFO - Found 20 files with duplicated chat_history path
+2025-04-10 18:59:31,306 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-180651.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f8ed-3fe84d2154e3e007658207e9;73ab81c7-35e8-46ae-92a1-667dac29201b)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:59:33,846 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-180725.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f8f0-66056ea913dbdff1621a3e5c;3cc301c9-c3ce-4c3a-916f-60b221917742)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:59:36,390 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-180940.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f8f2-36854ccc08420cd35c1613b5;a7ca9aea-ac48-4ab6-b4bd-696ae751173a)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:59:38,943 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-181628.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f8f5-4901cbfb535be455015ceb14;39923c77-0144-4d89-a830-8f80a2456e0b)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:59:41,452 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-181730.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f8f7-1eabdc675e3f49fe21c60749;88a17e2b-011e-4440-b6c8-54da2f298b03)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:59:44,025 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-184103.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f8fa-737ad0103e15fcc177b32fd4;25d06267-14df-4e02-aca0-c56fea86c85b)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:59:46,563 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-184613.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f8fd-070c52d76345eac007a8ddaf;d5d3f452-f76a-44c8-a439-96a47bc07e77)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:59:49,073 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-184705.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f8ff-646983a842f1edcd584b57cb;3f66f58c-621e-4842-be44-72c9d1c0fdde)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:59:51,615 - __main__ - ERROR - Error processing file chat_history/chat_history\20250404-185243.json: 429 Client Error: Too Many Requests for url: https://huggingface.co/api/datasets/Rulga/status-law-knowledge-base/commit/main (Request ID: Root=1-67f7f902-3dafd5c579a831d05562a7f0;a67f1694-a090-4590-b015-cb5b6f8a91ae)
+
+You have been rate-limited; you can retry this action in about 1 hour. If you're a new user, your limits will raise progressively over time. Get in touch with us at website@huggingface.co if you need access now.
+2025-04-10 18:59:52,416 - __main__ - INFO - 
+Received keyboard interrupt, stopping gracefully...
+2025-04-10 18:59:55,429 - __main__ - INFO - Process stopped by user

tools/repair_dataset_structure.py

@@ -0,0 +1,211 @@
+import logging
+import time
+import sys
+import os
+from pathlib import Path
+from dotenv import load_dotenv
+
+# Загружаем переменные окружения из .env файла
+load_dotenv()
+
+# Добавляем корневую директорию проекта в PYTHONPATH
+root_dir = str(Path(__file__).parent.parent)
+sys.path.append(root_dir)
+
+from huggingface_hub import HfApi
+from config.settings import (
+    DATASET_ID,
+    DATASET_CHAT_HISTORY_PATH,
+    HF_TOKEN
+)
+
+# Настройка логирования
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler("repair_dataset_structure.log", encoding='utf-8'),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+def repair_chat_files_structure():
+    """
+    Move misplaced chat files from root to existing chat_history directory
+    """
+    try:
+        api = HfApi(token=HF_TOKEN)
+        
+        # Получаем список всех файлов в датасете
+        files = api.list_repo_files(
+            repo_id=DATASET_ID,
+            repo_type="dataset"
+        )
+        
+        # Находим только файлы чата в корневой директории (без пути)
+        misplaced_files = [
+            f for f in files 
+            if f.endswith('.json') and 
+            '/' not in f and  # только файлы в корне
+            '-' in f  # характерный признак файлов чата (timestamp)
+        ]
+        
+        logger.info(f"Found {len(misplaced_files)} misplaced chat files")
+        
+        moved_count = 0
+        error_count = 0
+        
+        for file_path in misplaced_files:
+            try:
+                # Проверяем флаг остановки
+                if hasattr(repair_chat_files_structure, 'stop_flag') and repair_chat_files_structure.stop_flag:
+                    logger.info("Stopping process...")
+                    break
+
+                # Добавляем задержку между операциями
+                time.sleep(2)
+                
+                # Скачиваем содержимое файла
+                file_content = api.hf_hub_download(
+                    repo_id=DATASET_ID,
+                    filename=file_path,
+                    repo_type="dataset"
+                )
+                
+                # Перемещаем в существующую chat_history директорию
+                new_path = f"chat_history/{file_path}"
+                
+                # Загружаем файл в chat_history
+                with open(file_content, 'rb') as f:
+                    api.upload_file(
+                        path_or_fileobj=f,
+                        path_in_repo=new_path,
+                        repo_id=DATASET_ID,
+                        repo_type="dataset"
+                    )
+                
+                # Удаляем файл из корневой директории
+                api.delete_file(
+                    path_in_repo=file_path,
+                    repo_id=DATASET_ID,
+                    repo_type="dataset"
+                )
+                
+                logger.info(f"Moved {file_path} to {new_path}")
+                moved_count += 1
+                
+            except Exception as e:
+                logger.error(f"Error processing file {file_path}: {str(e)}")
+                error_count += 1
+                continue
+        
+        logger.info(f"Successfully moved {moved_count} files from root to chat_history")
+        if error_count > 0:
+            logger.warning(f"Failed to process {error_count} files")
+            
+    except Exception as e:
+        logger.error(f"Error accessing dataset: {str(e)}")
+
+def fix_duplicated_paths():
+    """
+    Fix duplicated chat_history paths in filenames
+    """
+    try:
+        api = HfApi(token=HF_TOKEN)
+        
+        # Получаем только файлы из папки chat_history с дублированным путем
+        wrong_paths = [
+            f for f in api.list_repo_files(
+                repo_id=DATASET_ID,
+                repo_type="dataset"
+            )
+            if f.startswith('chat_history/') and 
+            f.endswith('.json') and
+            'chat_history\\' in f  # ищем файлы с Windows-путем в имени
+        ]
+        
+        logger.info(f"Found {len(wrong_paths)} files with duplicated chat_history path")
+        
+        fixed_count = 0
+        error_count = 0
+        
+        for file_path in wrong_paths:
+            try:
+                # Проверяем флаг остановки
+                if hasattr(fix_duplicated_paths, 'stop_flag') and fix_duplicated_paths.stop_flag:
+                    logger.info("Stopping process...")
+                    break
+
+                # Добавляем задержку между операциями
+                time.sleep(2)
+                
+                # Скачиваем содержимое файла
+                file_content = api.hf_hub_download(
+                    repo_id=DATASET_ID,
+                    filename=file_path,
+                    repo_type="dataset"
+                )
+                
+                # Создаем правильный путь
+                filename = os.path.basename(file_path).replace('chat_history\\', '')
+                new_path = f"chat_history/{filename}"
+                
+                # Загружаем файл с правильным путем
+                with open(file_content, 'rb') as f:
+                    api.upload_file(
+                        path_or_fileobj=f,
+                        path_in_repo=new_path,
+                        repo_id=DATASET_ID,
+                        repo_type="dataset"
+                    )
+                
+                # Удаляем файл со старым путем
+                api.delete_file(
+                    path_in_repo=file_path,
+                    repo_id=DATASET_ID,
+                    repo_type="dataset"
+                )
+                
+                logger.info(f"Renamed {file_path} to {new_path}")
+                fixed_count += 1
+                
+            except Exception as e:
+                logger.error(f"Error processing file {file_path}: {str(e)}")
+                error_count += 1
+                continue
+        
+        logger.info(f"Successfully renamed {fixed_count} files")
+        if error_count > 0:
+            logger.warning(f"Failed to process {error_count} files")
+            
+    except Exception as e:
+        logger.error(f"Error accessing dataset: {str(e)}")
+
+if __name__ == "__main__":
+    try:
+        logger.info("=== Starting Dataset Structure Repair ===")
+        logger.info(f"Dataset ID: {DATASET_ID}")
+        
+        # Сначала перемещаем файлы из корня
+        #repair_chat_files_structure()
+        
+        # Затем исправляем пути
+        logger.info("=== Starting Path Fix ===")
+        fix_duplicated_paths()
+        
+        logger.info("=== Repair Complete ===")
+    except KeyboardInterrupt:
+        logger.info("\nReceived keyboard interrupt, stopping gracefully...")
+        repair_chat_files_structure.stop_flag = True
+        fix_duplicated_paths.stop_flag = True
+        time.sleep(3)
+        logger.info("Process stopped by user")
+    except Exception as e:
+        logger.error(f"Unexpected error: {str(e)}")
+
+
+
+
+
+