1.61 +decluster

app.py

@@ -2,7 +2,7 @@ import gradio as gr
 import spaces
 import pandas as pd
 import torch
-from transformers import pipeline, AutoModelForSeq2SeqLM, AutoTokenizer
+from transformers import pipeline, AutoModelForSeq2SeqLM, AutoTokenizer, AutoModel
 import plotly.graph_objects as go
 import logging
 import io
@@ -14,7 +14,10 @@ from langchain_openai import ChatOpenAI
 from langchain.prompts import PromptTemplate
 from openpyxl import load_workbook
 from openpyxl.utils.dataframe import dataframe_to_rows
-
+import torch.nn.functional as F
+import numpy as np
+import logging
+from typing import List, Set, Tuple
 
 def fuzzy_deduplicate(df, column, threshold=55):
     """Deduplicate rows based on fuzzy matching of text content"""
@@ -731,7 +734,7 @@ def create_interface():
         # Create state for file data
         current_file = gr.State(None)
         
-        gr.Markdown("# AI-анализ мониторинга новостей v.1.60 + добавка")
+        gr.Markdown("# AI-анализ мониторинга новостей v.1.61 + добавка")
         
         with gr.Row():
             file_input = gr.File(
@@ -793,12 +796,14 @@ def create_interface():
             control.request_stop()
             return "Остановка обработки..."
                 
+        
         @spaces.GPU(duration=300)
         def process_and_download(file_bytes):
             if file_bytes is None:
                 gr.Warning("Пожалуйста, загрузите файл")
                 return (pd.DataFrame(), None, None, None, "Ожидание файла...", "")
-                
+            
+            detector = None    
             try:
                 file_obj = io.BytesIO(file_bytes)
                 logger.info("File loaded into BytesIO successfully")
@@ -906,14 +911,7 @@ def create_interface():
                 error_msg = f"Ошибка анализа: {str(e)}"
                 logger.error(error_msg)
                 gr.Error(error_msg)
-                return (
-                    pd.DataFrame(),
-                    None,
-                    None,
-                    None,
-                    error_msg,
-                    ""
-                )
+                return (pd.DataFrame(), None, None, None, error_msg, "")
             finally:
                 if detector:
                     detector.cleanup()

requirements.txt

@@ -23,4 +23,8 @@ plotly
 datetime
 langchain_openai
 langchain
-langchain-community
+langchain-community
+numpy
+spacy
+pymorphy2
+https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.0/en_core_web_sm-3.7.0.tar.gz