Spaces:
Sleeping
Sleeping
Commit
·
1c69559
1
Parent(s):
83c6d36
Fixed HF_REPO_ID + Added force_download=True + Enhanced status messages
Browse files- converters/converter.py +24 -32
converters/converter.py
CHANGED
|
@@ -26,7 +26,6 @@ def process_uploaded_file(file, file_type):
|
|
| 26 |
shutil.copy(source_path, file_path)
|
| 27 |
else:
|
| 28 |
file_path = source_path
|
| 29 |
-
|
| 30 |
original_size_bytes = os.path.getsize(file_path)
|
| 31 |
original_size_mb = original_size_bytes / (1024 * 1024)
|
| 32 |
|
|
@@ -34,32 +33,13 @@ def process_uploaded_file(file, file_type):
|
|
| 34 |
status_info.append(f"📁 Исходный файл: {filename}")
|
| 35 |
status_info.append(f"📦 Размер файла: {original_size_mb:.2f} МБ ({original_size_bytes:,} байт)")
|
| 36 |
|
| 37 |
-
|
| 38 |
-
if file_type == "JSON документ":
|
| 39 |
-
target_dir = JSON_FILES_DIR
|
| 40 |
-
upload_file = file_path
|
| 41 |
-
|
| 42 |
-
try:
|
| 43 |
-
with open(upload_file, 'r', encoding='utf-8') as f:
|
| 44 |
-
json_data = json.load(f)
|
| 45 |
-
|
| 46 |
-
# Count sections
|
| 47 |
-
section_count = len(json_data.get('sections', []))
|
| 48 |
-
status_info.append(f"📝 Разделов в документе: {section_count}")
|
| 49 |
-
status_info.append(f"📄 ID документа: {json_data.get('document_metadata', {}).get('document_id', 'N/A')}")
|
| 50 |
-
|
| 51 |
-
except Exception as e:
|
| 52 |
-
return f"❌ Ошибка чтения JSON: {str(e)}"
|
| 53 |
-
|
| 54 |
-
status_info.append(f"📤 Загружен как: {filename}")
|
| 55 |
-
status_info.append(f"📂 Целевая директория: {target_dir}")
|
| 56 |
-
|
| 57 |
-
elif file_type == "Таблица":
|
| 58 |
target_dir = TABLE_DATA_DIR
|
| 59 |
if filename.endswith(('.xlsx', '.xls')):
|
| 60 |
json_path = convert_single_excel_to_json(file_path, temp_dir)
|
| 61 |
upload_file = json_path
|
| 62 |
|
|
|
|
| 63 |
processed_size_bytes = os.path.getsize(json_path)
|
| 64 |
processed_size_mb = processed_size_bytes / (1024 * 1024)
|
| 65 |
|
|
@@ -73,11 +53,9 @@ def process_uploaded_file(file, file_type):
|
|
| 73 |
status_info.append(f"📝 Всего строк данных: {total_rows:,}")
|
| 74 |
status_info.append(f"💾 Размер после обработки: {processed_size_mb:.2f} МБ")
|
| 75 |
status_info.append(f"📤 Загружен как: {os.path.basename(json_path)}")
|
| 76 |
-
status_info.append(f"📂 Целевая директория: {target_dir}")
|
| 77 |
else:
|
| 78 |
upload_file = file_path
|
| 79 |
status_info.append(f"📤 Загружен как: {filename}")
|
| 80 |
-
status_info.append(f"📂 Целевая директория: {target_dir}")
|
| 81 |
|
| 82 |
elif file_type == "Изображение (метаданные)":
|
| 83 |
target_dir = IMAGE_DATA_DIR
|
|
@@ -85,6 +63,7 @@ def process_uploaded_file(file, file_type):
|
|
| 85 |
csv_path = convert_single_excel_to_csv(file_path, temp_dir)
|
| 86 |
upload_file = csv_path
|
| 87 |
|
|
|
|
| 88 |
processed_size_bytes = os.path.getsize(csv_path)
|
| 89 |
processed_size_mb = processed_size_bytes / (1024 * 1024)
|
| 90 |
|
|
@@ -93,7 +72,6 @@ def process_uploaded_file(file, file_type):
|
|
| 93 |
status_info.append(f"📋 Колонок метаданных: {len(df.columns)}")
|
| 94 |
status_info.append(f"💾 Размер после обработки: {processed_size_mb:.2f} МБ")
|
| 95 |
status_info.append(f"📤 Загружен как: {os.path.basename(csv_path)}")
|
| 96 |
-
status_info.append(f"📂 Целевая директория: {target_dir}")
|
| 97 |
else:
|
| 98 |
upload_file = file_path
|
| 99 |
try:
|
|
@@ -103,17 +81,32 @@ def process_uploaded_file(file, file_type):
|
|
| 103 |
except:
|
| 104 |
pass
|
| 105 |
status_info.append(f"📤 Загружен как: {filename}")
|
| 106 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
|
| 108 |
# Загружаем на HuggingFace
|
| 109 |
-
|
| 110 |
-
log_message(f"Загрузка на HuggingFace: {upload_path}")
|
| 111 |
-
status_info.append(f"⬆️ Загрузка в репозиторий...")
|
| 112 |
-
|
| 113 |
api = HfApi()
|
| 114 |
api.upload_file(
|
| 115 |
path_or_fileobj=upload_file,
|
| 116 |
-
path_in_repo=
|
| 117 |
repo_id=HF_REPO_ID,
|
| 118 |
token=HF_TOKEN,
|
| 119 |
repo_type="dataset"
|
|
@@ -123,7 +116,6 @@ def process_uploaded_file(file, file_type):
|
|
| 123 |
|
| 124 |
result_message = f"✅ Файл успешно загружен и обработан\n\n"
|
| 125 |
result_message += "\n".join(status_info)
|
| 126 |
-
result_message += f"\n\n✅ Файл добавлен в: {upload_path}"
|
| 127 |
result_message += "\n\n⚠️ Нажмите кнопку 'Перезапустить систему' для применения изменений"
|
| 128 |
|
| 129 |
return result_message
|
|
|
|
| 26 |
shutil.copy(source_path, file_path)
|
| 27 |
else:
|
| 28 |
file_path = source_path
|
|
|
|
| 29 |
original_size_bytes = os.path.getsize(file_path)
|
| 30 |
original_size_mb = original_size_bytes / (1024 * 1024)
|
| 31 |
|
|
|
|
| 33 |
status_info.append(f"📁 Исходный файл: {filename}")
|
| 34 |
status_info.append(f"📦 Размер файла: {original_size_mb:.2f} МБ ({original_size_bytes:,} байт)")
|
| 35 |
|
| 36 |
+
if file_type == "Таблица":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
target_dir = TABLE_DATA_DIR
|
| 38 |
if filename.endswith(('.xlsx', '.xls')):
|
| 39 |
json_path = convert_single_excel_to_json(file_path, temp_dir)
|
| 40 |
upload_file = json_path
|
| 41 |
|
| 42 |
+
# Get processed file size
|
| 43 |
processed_size_bytes = os.path.getsize(json_path)
|
| 44 |
processed_size_mb = processed_size_bytes / (1024 * 1024)
|
| 45 |
|
|
|
|
| 53 |
status_info.append(f"📝 Всего строк данных: {total_rows:,}")
|
| 54 |
status_info.append(f"💾 Размер после обработки: {processed_size_mb:.2f} МБ")
|
| 55 |
status_info.append(f"📤 Загружен как: {os.path.basename(json_path)}")
|
|
|
|
| 56 |
else:
|
| 57 |
upload_file = file_path
|
| 58 |
status_info.append(f"📤 Загружен как: {filename}")
|
|
|
|
| 59 |
|
| 60 |
elif file_type == "Изображение (метаданные)":
|
| 61 |
target_dir = IMAGE_DATA_DIR
|
|
|
|
| 63 |
csv_path = convert_single_excel_to_csv(file_path, temp_dir)
|
| 64 |
upload_file = csv_path
|
| 65 |
|
| 66 |
+
# Get processed file size
|
| 67 |
processed_size_bytes = os.path.getsize(csv_path)
|
| 68 |
processed_size_mb = processed_size_bytes / (1024 * 1024)
|
| 69 |
|
|
|
|
| 72 |
status_info.append(f"📋 Колонок метаданных: {len(df.columns)}")
|
| 73 |
status_info.append(f"💾 Размер после обработки: {processed_size_mb:.2f} МБ")
|
| 74 |
status_info.append(f"📤 Загружен как: {os.path.basename(csv_path)}")
|
|
|
|
| 75 |
else:
|
| 76 |
upload_file = file_path
|
| 77 |
try:
|
|
|
|
| 81 |
except:
|
| 82 |
pass
|
| 83 |
status_info.append(f"📤 Загружен как: {filename}")
|
| 84 |
+
|
| 85 |
+
else: # JSON документ
|
| 86 |
+
target_dir = JSON_FILES_DIR
|
| 87 |
+
upload_file = file_path
|
| 88 |
+
|
| 89 |
+
try:
|
| 90 |
+
with open(upload_file, 'r', encoding='utf-8') as f:
|
| 91 |
+
json_data = json.load(f)
|
| 92 |
+
|
| 93 |
+
if isinstance(json_data, list):
|
| 94 |
+
status_info.append(f"📝 Документов в JSON: {len(json_data):,}")
|
| 95 |
+
elif isinstance(json_data, dict):
|
| 96 |
+
status_info.append(f"📝 JSON объект (словарь)")
|
| 97 |
+
# Count keys if it's structured data
|
| 98 |
+
if 'sheets' in json_data:
|
| 99 |
+
status_info.append(f"📊 Таблиц в документе: {len(json_data.get('sheets', []))}")
|
| 100 |
+
except:
|
| 101 |
+
pass
|
| 102 |
+
status_info.append(f"📤 Загружен как: {filename}")
|
| 103 |
|
| 104 |
# Загружаем на HuggingFace
|
| 105 |
+
log_message(f"Загрузка на HuggingFace: {target_dir}/{os.path.basename(upload_file)}")
|
|
|
|
|
|
|
|
|
|
| 106 |
api = HfApi()
|
| 107 |
api.upload_file(
|
| 108 |
path_or_fileobj=upload_file,
|
| 109 |
+
path_in_repo=f"{target_dir}/{os.path.basename(upload_file)}",
|
| 110 |
repo_id=HF_REPO_ID,
|
| 111 |
token=HF_TOKEN,
|
| 112 |
repo_type="dataset"
|
|
|
|
| 116 |
|
| 117 |
result_message = f"✅ Файл успешно загружен и обработан\n\n"
|
| 118 |
result_message += "\n".join(status_info)
|
|
|
|
| 119 |
result_message += "\n\n⚠️ Нажмите кнопку 'Перезапустить систему' для применения изменений"
|
| 120 |
|
| 121 |
return result_message
|