Update app.py

app.py

@@ -1,6 +1,7 @@
 import os
 import re
 from camel_tools.tokenizers.word import simple_word_tokenize
+from camel_tools.ner import NERecognizer
 import nltk
 import torch
 from collections import Counter
@@ -8,6 +9,8 @@ from transformers import pipeline, AutoModel, AutoTokenizer
 import PyPDF2
 import gradio as gr
 import openai
+from haystack.nodes import FARMReader
+from paddlenlp import Taskflow
 
 # تحميل وتفعيل الأدوات المطلوبة
 nltk.download('punkt')
@@ -15,42 +18,38 @@ nltk.download('punkt')
 # التحقق من توفر GPU واستخدامه
 device = 0 if torch.cuda.is_available() else -1
 
-# إعداد التوكنات
-openai.api_key = "sk-proj-62TDbO5KABSdkZaFPPD4T3BlbkFJkhqOYpHhL6OucTzNdWSU"
-
 # تحميل نماذج التحليل اللغوي
-analyzer = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english", device=device, use_auth_token=huggingface_token)
+analyzer = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english", device=device)
+
+# تحميل نموذج التعرف على الكيانات في camel_tools
+ner = NERecognizer.pretrained()
 
 # تحميل نماذج BERT، GPT2، ELECTRA، و AraBERT
-arabic_bert_tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-base-arabic", use_auth_token=huggingface_token)
-arabic_bert_model = AutoModel.from_pretrained("asafaya/bert-base-arabic", use_auth_token=huggingface_token)
+arabic_bert_tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-base-arabic")
+arabic_bert_model = AutoModel.from_pretrained("asafaya/bert-base-arabic")
 
-arabic_gpt2_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/aragpt2-base", use_auth_token=huggingface_token)
-arabic_gpt2_model = AutoModel.from_pretrained("aubmindlab/aragpt2-base", use_auth_token=huggingface_token)
+arabic_gpt2_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/aragpt2-base")
+arabic_gpt2_model = AutoModel.from_pretrained("aubmindlab/aragpt2-base")
 
-arabic_electra_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/araelectra-base-discriminator", use_auth_token=huggingface_token)
-arabic_electra_model = AutoModel.from_pretrained("aubmindlab/araelectra-base-discriminator", use_auth_token=huggingface_token)
+arabic_electra_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/araelectra-base-discriminator")
+arabic_electra_model = AutoModel.from_pretrained("aubmindlab/araelectra-base-discriminator")
 
-arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv02", use_auth_token=huggingface_token)
-arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv02", use_auth_token=huggingface_token)
+arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv02")
+arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv02")
 
-aragpt2_mega_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/aragpt2-mega", use_auth_token=huggingface_token)
-aragpt2_mega_model = AutoModel.from_pretrained("aubmindlab/aragpt2-mega", use_auth_token=huggingface_token)
+# إعداد OpenAI API
+openai.api_key = os.getenv("sk-proj-62TDbO5KABSdkZaFPPD4T3BlbkFJkhqOYpHhL6OucTzNdWSU")
 
-xlm_roberta_tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-large", use_auth_token=huggingface_token)
-xlm_roberta_model = AutoModel.from_pretrained("xlm-roberta-large", use_auth_token=huggingface_token)
+# إعداد farm-haystack
+reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2")
 
-m2m100_tokenizer = AutoTokenizer.from_pretrained("facebook/m2m100_418M", use_auth_token=huggingface_token)
-m2m100_model = AutoModel.from_pretrained("facebook/m2m100_418M", use_auth_token=huggingface_token)
+# إعداد paddlenlp
+ner_task = Taskflow("ner")
 
-# دالة لتحليل النص باستخدام arabert-ner من transformers
+# دالة لتحليل النص باستخدام camel_tools
 def camel_ner_analysis(text):
-    tokenizer = AutoTokenizer.from_pretrained("camel-ai/arabert-ner", use_auth_token=huggingface_token)
-    model = AutoModel.from_pretrained("camel-ai/arabert-ner", use_auth_token=huggingface_token)
     tokens = simple_word_tokenize(text)
-    inputs = tokenizer(tokens, return_tensors="pt", is_split_into_words=True)
-    outputs = model(**inputs)
-    entities = outputs.logits.argmax(dim=-1).squeeze().tolist()
+    entities = ner.predict(tokens)
     entity_dict = {"PERSON": [], "LOC": [], "ORG": [], "DATE": []}
     for token, tag in zip(tokens, entities):
         if tag in entity_dict:
@@ -134,7 +133,7 @@ def extract_dialogues(text):
 # دالة لتحليل النصوص واستخراج المعلومات وحفظ النتائج
 def analyze_and_complete(file_paths):
     results = []
-    output_directory = "/Volumes/CLOCKWORK T/clockworkspace/first pro/out"
+    output_directory = os.getenv("SPACE_DIR", "/app/output")
     
     for file_path in file_paths:
         if file_path.endswith(".pdf"):