Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -11,21 +11,31 @@ def paraphrase_text(text):
|
|
| 11 |
input_ids = tokenizer.encode(input_text, return_tensors="pt", truncation=True)
|
| 12 |
output_ids = model.generate(
|
| 13 |
input_ids,
|
| 14 |
-
max_length=
|
| 15 |
do_sample=True,
|
| 16 |
top_k=120,
|
| 17 |
top_p=0.95,
|
| 18 |
-
temperature=1.
|
|
|
|
| 19 |
)
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
-
def chunk_text(text, max_sentences=
|
| 23 |
sentences = re.split(r'(?<=[.!?]) +', text.strip())
|
| 24 |
return [' '.join(sentences[i:i+max_sentences]) for i in range(0, len(sentences), max_sentences)]
|
| 25 |
|
| 26 |
def full_article_paraphrase(text):
|
| 27 |
chunks = chunk_text(text)
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
def extract_text_from_docx(file_obj):
|
| 31 |
from docx import Document
|
|
|
|
| 11 |
input_ids = tokenizer.encode(input_text, return_tensors="pt", truncation=True)
|
| 12 |
output_ids = model.generate(
|
| 13 |
input_ids,
|
| 14 |
+
max_length=1024, # allow longer output
|
| 15 |
do_sample=True,
|
| 16 |
top_k=120,
|
| 17 |
top_p=0.95,
|
| 18 |
+
temperature=1.2, # moderate creativity
|
| 19 |
+
early_stopping=True
|
| 20 |
)
|
| 21 |
+
paraphrased = tokenizer.decode(output_ids[0], skip_special_tokens=True)
|
| 22 |
+
# Fix spacing around punctuation
|
| 23 |
+
paraphrased = re.sub(r'\s+([.,!?;:])', r'\1', paraphrased)
|
| 24 |
+
paraphrased = re.sub(r'([(\["])\s+', r'\1', paraphrased)
|
| 25 |
+
paraphrased = re.sub(r'\s+([)\]"])', r'\1', paraphrased)
|
| 26 |
+
return paraphrased.strip()
|
| 27 |
|
| 28 |
+
def chunk_text(text, max_sentences=6): # bigger chunks for better flow
|
| 29 |
sentences = re.split(r'(?<=[.!?]) +', text.strip())
|
| 30 |
return [' '.join(sentences[i:i+max_sentences]) for i in range(0, len(sentences), max_sentences)]
|
| 31 |
|
| 32 |
def full_article_paraphrase(text):
|
| 33 |
chunks = chunk_text(text)
|
| 34 |
+
paraphrased_chunks = []
|
| 35 |
+
for chunk in chunks:
|
| 36 |
+
if chunk.strip():
|
| 37 |
+
paraphrased_chunks.append(paraphrase_text(chunk.strip()))
|
| 38 |
+
return "\n\n".join(paraphrased_chunks)
|
| 39 |
|
| 40 |
def extract_text_from_docx(file_obj):
|
| 41 |
from docx import Document
|