1.8

app.py

@@ -27,6 +27,25 @@ class NewsProcessor:
         self.model = AutoModel.from_pretrained('sentence-transformers/paraphrase-multilingual-mpnet-base-v2')
         self.similarity_threshold = similarity_threshold
         self.time_threshold = time_threshold
+    
+    def mean_pooling(self, model_output, attention_mask):
+        token_embeddings = model_output[0]
+        input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+        return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+    def encode_text(self, text):
+        # Convert text to string and handle NaN values
+        if pd.isna(text):
+            text = ""
+        else:
+            text = str(text)
+            
+        encoded_input = self.tokenizer(text, padding=True, truncation=True, max_length=512, return_tensors='pt')
+        with torch.no_grad():
+            model_output = self.model(**encoded_input)
+        sentence_embeddings = self.mean_pooling(model_output, encoded_input['attention_mask'])
+        return F.normalize(sentence_embeddings[0], p=2, dim=0).numpy()
+    
 
     def get_company_variants(self, company_name: str) -> Set[str]:
         """Generate morphological variants of company name."""
@@ -302,7 +321,7 @@ def create_download_link(df: pd.DataFrame, filename: str) -> str:
     return f'<a href="data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,{b64}" download="{filename}">Download {filename}</a>'
 
 def main():
-    st.title("кластеризуем новости v.1.7")
+    st.title("кластеризуем новости v.1.8")
     st.write("Upload Excel file with columns: company, datetime, text")
     
     uploaded_file = st.file_uploader("Choose Excel file", type=['xlsx'])