| | import os |
| |
|
| | def process_file(file_path): |
| | with open(file_path, 'r', encoding='utf-8-sig') as f: |
| | text = f.read() |
| | assert text |
| | return text, str(file_path).split('.')[0] |
| |
|
| |
|
| | def process_folder_recursive(folder_path): |
| | all_messages = [] |
| | for file in os.listdir(folder_path): |
| | file_path = os.path.join(folder_path, file) |
| | if os.path.isfile(file_path): |
| | all_messages.append(process_file(file_path)) |
| | else: |
| | all_messages += process_folder_recursive(file_path) |
| | return all_messages |
| |
|
| |
|
| | def load_and_process_data() -> list[dict]: |
| | """Загрузка и предобработка данных из JSON файлов""" |
| | all_messages = process_folder_recursive('texts') |
| | return [x[0] for x in all_messages], [x[1][:-3] for x in all_messages] |
| |
|
