Yermek68 commited on
Commit
5a1bb5b
·
verified ·
1 Parent(s): a669d98

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +63 -93
app.py CHANGED
@@ -8,169 +8,139 @@ from fpdf import FPDF
8
  from langdetect import detect
9
  import urllib.request
10
 
11
-
12
- # === 🗂️ Инициализация ===
13
  os.makedirs("/app/models", exist_ok=True)
14
  FONT_PATH = "DejaVuSans.ttf"
15
 
16
  if not os.path.exists(FONT_PATH):
17
- print("⬇️ Загружаю шрифт DejaVuSans.ttf ...")
18
  urllib.request.urlretrieve(
19
  "https://github.com/dejavu-fonts/dejavu-fonts/raw/master/ttf/DejaVuSans.ttf",
20
  FONT_PATH
21
  )
22
 
 
 
 
 
23
 
24
- # === ⚙️ Модели ===
25
- def load_model(task, model_name):
26
- print(f"🔹 Загружается модель: {model_name}")
27
- return pipeline(task, model=model_name, cache_dir="/app/models")
28
-
29
- summarizers = {
30
  "en": load_model("summarization", "facebook/bart-large-cnn"),
31
  "ru": load_model("summarization", "IlyaGusev/mbart_ru_sum_gazeta"),
32
- "kz": load_model("summarization", "csebuetnlp/mT5_multilingual_XLSum")
33
  }
34
 
35
-
36
- # === 📄 Чтение файлов ===
37
  def read_file(file):
38
  if not file:
39
  return ""
40
- filename = file.name.lower()
41
  text = ""
42
  try:
43
- if filename.endswith(".pdf"):
44
  with pdfplumber.open(file.name) as pdf:
45
  for page in pdf.pages:
46
- if page.extract_text():
47
- text += page.extract_text() + "\n"
48
- elif filename.endswith(".docx"):
 
49
  text = docx2txt.process(file.name)
50
  else:
51
  text = file.read().decode("utf-8", errors="ignore")
52
  except Exception as e:
53
- return f"⚠️ Ошибка при чтении файла: {e}"
54
  return text.strip()
55
 
56
-
57
- # === 🌐 Определение языка ===
58
  def detect_language(text):
59
  try:
60
  lang = detect(text)
61
  if lang.startswith("ru"):
62
  return "ru"
63
- elif lang.startswith("kk") or any(x in text for x in "әіңғүұқөһ"):
64
  return "kz"
65
  else:
66
  return "en"
67
  except:
68
  return "en"
69
 
70
-
71
- # === 🧠 Суммаризация ===
72
  def summarize_text(text):
73
  if not text or len(text) < 50:
74
- return "⚠️ Недостаточно текста для анализа.", "❌", "❌", 0, 0, "❌"
75
 
76
  lang = detect_language(text)
77
- model = summarizers.get(lang, summarizers["en"])
78
  flags = {"ru": "🇷🇺 Русский", "kz": "🇰🇿 Қазақ тілі", "en": "🇬🇧 English"}
79
  lang_label = flags.get(lang, "🌍 Unknown")
80
- model_label = model.model.name_or_path if hasattr(model.model, "name_or_path") else "Custom"
81
 
82
  chunk_size = 2500
83
  overlap = 200
84
  summaries = []
85
 
86
  for i in range(0, len(text), chunk_size - overlap):
87
- chunk = text[i:i + chunk_size]
88
  try:
89
- result = model(chunk, max_length=180, min_length=40, do_sample=False)
90
- summaries.append(result[0]['summary_text'])
91
  except Exception as e:
92
  summaries.append(f"[Ошибка в части {len(summaries)+1}: {e}]")
93
 
94
  summary = "\n\n".join(summaries).strip()
95
  src_len = len(text)
96
  sum_len = len(summary)
97
- compression = round(100 * (1 - sum_len / src_len), 1) if src_len > 0 else 0
98
-
99
- return summary, lang_label, model_label, src_len, sum_len, f"{compression}%"
100
-
101
-
102
- # === 💾 Сохранение ===
103
- def save_summary_as_txt(summary_text):
104
- path = "summary.txt"
105
- with open(path, "w", encoding="utf-8") as f:
106
- f.write(summary_text)
107
- return path
108
-
109
- def save_summary_as_docx(summary_text):
110
- path = "summary.docx"
111
- doc = Document()
112
- doc.add_heading("Резюме документа", level=1)
113
- doc.add_paragraph(summary_text)
114
- doc.save(path)
115
- return path
116
-
117
- def save_summary_as_pdf(summary_text):
118
- path = "summary.pdf"
119
- pdf = FPDF()
120
- pdf.add_page()
121
- pdf.add_font('DejaVu', '', FONT_PATH, uni=True)
122
- pdf.set_font('DejaVu', '', 12)
123
- pdf.multi_cell(0, 10, summary_text)
124
- pdf.output(path)
125
- return path
126
-
127
-
128
- # === 🚀 Основная функция ===
129
- def summarize_file(file):
130
  text = read_file(file)
131
  if text.startswith("⚠️"):
132
  return text, "❌", "❌", 0, 0, "❌", None, None, None
133
 
134
- with gr.Progress(track_tqdm=True) as progress:
135
- progress(0, desc="🧠 Анализ текста...")
136
- summary, lang_label, model_label, src_len, sum_len, compression = summarize_text(text)
137
- progress(1, desc="✅ Готово!")
138
-
139
- txt_path = save_summary_as_txt(summary)
140
- docx_path = save_summary_as_docx(summary)
141
- pdf_path = save_summary_as_pdf(summary)
142
 
143
- return summary, lang_label, model_label, src_len, sum_len, compression, txt_path, docx_path, pdf_path
144
-
145
-
146
- # === 🧩 Интерфейс (универсальный для любых версий Gradio 4.4+) ===
147
  with gr.Blocks() as demo:
148
- gr.Markdown(
149
- "<h2 style='text-align:center'>🧠 Eroha Summarizer PRO</h2>"
150
- "<p style='text-align:center'>Определяет язык (🇷🇺 / 🇰🇿 / 🇬🇧), создаёт краткое резюме и сохраняет в TXT, DOCX, PDF</p>"
151
- )
152
-
153
  file_input = gr.File(label="📂 Загрузите документ (.pdf, .docx, .txt)")
154
  run_btn = gr.Button("🔍 Сгенерировать резюме", variant="primary")
155
-
156
- summary_output = gr.Textbox(label="🧾 Краткое резюме", lines=10)
157
- lang_output = gr.Textbox(label="🌍 Определённый язык")
158
- model_output = gr.Textbox(label="🧠 Используемая модель")
159
  src_len = gr.Number(label="📄 Длина исходного текста")
160
  sum_len = gr.Number(label="📝 Длина резюме")
161
- compression = gr.Textbox(label="📉 Степень сокращения")
162
- txt_file = gr.File(label="📄 TXT файл")
163
- docx_file = gr.File(label="📘 DOCX файл")
164
- pdf_file = gr.File(label="📕 PDF файл")
165
 
166
  run_btn.click(
167
  summarize_file,
168
- inputs=[file_input],
169
- outputs=[
170
- summary_output, lang_output, model_output,
171
- src_len, sum_len, compression,
172
- txt_file, docx_file, pdf_file
173
- ]
174
  )
175
 
176
  demo.launch(server_name="0.0.0.0", server_port=7860)
 
8
  from langdetect import detect
9
  import urllib.request
10
 
11
+ # === Настройки ===
 
12
  os.makedirs("/app/models", exist_ok=True)
13
  FONT_PATH = "DejaVuSans.ttf"
14
 
15
  if not os.path.exists(FONT_PATH):
 
16
  urllib.request.urlretrieve(
17
  "https://github.com/dejavu-fonts/dejavu-fonts/raw/master/ttf/DejaVuSans.ttf",
18
  FONT_PATH
19
  )
20
 
21
+ # === Модели ===
22
+ def load_model(task, name):
23
+ print(f"🔹 Загружается модель: {name}")
24
+ return pipeline(task, model=name, cache_dir="/app/models")
25
 
26
+ models = {
 
 
 
 
 
27
  "en": load_model("summarization", "facebook/bart-large-cnn"),
28
  "ru": load_model("summarization", "IlyaGusev/mbart_ru_sum_gazeta"),
29
+ "kz": load_model("summarization", "csebuetnlp/mT5_multilingual_XLSum"),
30
  }
31
 
32
+ # === Чтение файлов ===
 
33
  def read_file(file):
34
  if not file:
35
  return ""
36
+ path = file.name.lower()
37
  text = ""
38
  try:
39
+ if path.endswith(".pdf"):
40
  with pdfplumber.open(file.name) as pdf:
41
  for page in pdf.pages:
42
+ t = page.extract_text()
43
+ if t:
44
+ text += t + "\n"
45
+ elif path.endswith(".docx"):
46
  text = docx2txt.process(file.name)
47
  else:
48
  text = file.read().decode("utf-8", errors="ignore")
49
  except Exception as e:
50
+ return f"⚠️ Ошибка при чтении: {e}"
51
  return text.strip()
52
 
53
+ # === Определение языка ===
 
54
  def detect_language(text):
55
  try:
56
  lang = detect(text)
57
  if lang.startswith("ru"):
58
  return "ru"
59
+ elif lang.startswith("kk") or any(c in text for c in "әіңғүұқөһ"):
60
  return "kz"
61
  else:
62
  return "en"
63
  except:
64
  return "en"
65
 
66
+ # === Суммаризация ===
 
67
  def summarize_text(text):
68
  if not text or len(text) < 50:
69
+ return "⚠️ Недостаточно текста", "❌", "❌", 0, 0, "❌"
70
 
71
  lang = detect_language(text)
72
+ model = models.get(lang, models["en"])
73
  flags = {"ru": "🇷🇺 Русский", "kz": "🇰🇿 Қазақ тілі", "en": "🇬🇧 English"}
74
  lang_label = flags.get(lang, "🌍 Unknown")
75
+ model_label = getattr(model.model, "name_or_path", "Custom")
76
 
77
  chunk_size = 2500
78
  overlap = 200
79
  summaries = []
80
 
81
  for i in range(0, len(text), chunk_size - overlap):
82
+ chunk = text[i:i+chunk_size]
83
  try:
84
+ res = model(chunk, max_length=180, min_length=40, do_sample=False)
85
+ summaries.append(res[0]['summary_text'])
86
  except Exception as e:
87
  summaries.append(f"[Ошибка в части {len(summaries)+1}: {e}]")
88
 
89
  summary = "\n\n".join(summaries).strip()
90
  src_len = len(text)
91
  sum_len = len(summary)
92
+ comp = f"{round(100*(1 - sum_len/src_len),1)}%" if src_len else "0%"
93
+ return summary, lang_label, model_label, src_len, sum_len, comp
94
+
95
+ # === Сохранение ===
96
+ def save_summary(summary):
97
+ txt, docx, pdf = "summary.txt", "summary.docx", "summary.pdf"
98
+ with open(txt, "w", encoding="utf-8") as f:
99
+ f.write(summary)
100
+ d = Document()
101
+ d.add_heading("Резюме", 0)
102
+ d.add_paragraph(summary)
103
+ d.save(docx)
104
+ pdf_doc = FPDF()
105
+ pdf_doc.add_page()
106
+ pdf_doc.add_font('DejaVu', '', FONT_PATH, uni=True)
107
+ pdf_doc.set_font('DejaVu', '', 12)
108
+ pdf_doc.multi_cell(0, 10, summary)
109
+ pdf_doc.output(pdf)
110
+ return txt, docx, pdf
111
+
112
+ # === Основная функция ===
113
+ def summarize_file(file, progress=gr.Progress()):
 
 
 
 
 
 
 
 
 
 
 
114
  text = read_file(file)
115
  if text.startswith("⚠️"):
116
  return text, "❌", "❌", 0, 0, "❌", None, None, None
117
 
118
+ progress(0.2, "🧠 Анализ текста...")
119
+ summary, lang, model, src, summ, comp = summarize_text(text)
120
+ progress(0.8, "📄 Сохранение файлов...")
121
+ txt, docx, pdf = save_summary(summary)
122
+ progress(1, "✅ Готово!")
123
+ return summary, lang, model, src, summ, comp, txt, docx, pdf
 
 
124
 
125
+ # === Интерфейс ===
 
 
 
126
  with gr.Blocks() as demo:
127
+ gr.Markdown("<h2 style='text-align:center'>🧠 Eroha Summarizer PRO</h2>")
 
 
 
 
128
  file_input = gr.File(label="📂 Загрузите документ (.pdf, .docx, .txt)")
129
  run_btn = gr.Button("🔍 Сгенерировать резюме", variant="primary")
130
+ summary_out = gr.Textbox(label="🧾 Краткое резюме", lines=10)
131
+ lang_out = gr.Textbox(label="🌍 Определённый язык")
132
+ model_out = gr.Textbox(label="🧠 Используемая модель")
 
133
  src_len = gr.Number(label="📄 Длина исходного текста")
134
  sum_len = gr.Number(label="📝 Длина резюме")
135
+ comp = gr.Textbox(label="📉 Степень сокращения")
136
+ txt = gr.File(label="📄 TXT")
137
+ docx = gr.File(label="📘 DOCX")
138
+ pdf = gr.File(label="📕 PDF")
139
 
140
  run_btn.click(
141
  summarize_file,
142
+ inputs=file_input,
143
+ outputs=[summary_out, lang_out, model_out, src_len, sum_len, comp, txt, docx, pdf],
 
 
 
 
144
  )
145
 
146
  demo.launch(server_name="0.0.0.0", server_port=7860)