VictorM-Coder commited on
Commit
6b6ee15
·
verified ·
1 Parent(s): f5ee1bc

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +15 -5
app.py CHANGED
@@ -11,21 +11,31 @@ def paraphrase_text(text):
11
  input_ids = tokenizer.encode(input_text, return_tensors="pt", truncation=True)
12
  output_ids = model.generate(
13
  input_ids,
14
- max_length=256,
15
  do_sample=True,
16
  top_k=120,
17
  top_p=0.95,
18
- temperature=1.5
 
19
  )
20
- return tokenizer.decode(output_ids[0], skip_special_tokens=True)
 
 
 
 
 
21
 
22
- def chunk_text(text, max_sentences=4):
23
  sentences = re.split(r'(?<=[.!?]) +', text.strip())
24
  return [' '.join(sentences[i:i+max_sentences]) for i in range(0, len(sentences), max_sentences)]
25
 
26
  def full_article_paraphrase(text):
27
  chunks = chunk_text(text)
28
- return "\n\n".join(paraphrase_text(chunk.strip()) for chunk in chunks if chunk.strip())
 
 
 
 
29
 
30
  def extract_text_from_docx(file_obj):
31
  from docx import Document
 
11
  input_ids = tokenizer.encode(input_text, return_tensors="pt", truncation=True)
12
  output_ids = model.generate(
13
  input_ids,
14
+ max_length=1024, # allow longer output
15
  do_sample=True,
16
  top_k=120,
17
  top_p=0.95,
18
+ temperature=1.2, # moderate creativity
19
+ early_stopping=True
20
  )
21
+ paraphrased = tokenizer.decode(output_ids[0], skip_special_tokens=True)
22
+ # Fix spacing around punctuation
23
+ paraphrased = re.sub(r'\s+([.,!?;:])', r'\1', paraphrased)
24
+ paraphrased = re.sub(r'([(\["])\s+', r'\1', paraphrased)
25
+ paraphrased = re.sub(r'\s+([)\]"])', r'\1', paraphrased)
26
+ return paraphrased.strip()
27
 
28
+ def chunk_text(text, max_sentences=6): # bigger chunks for better flow
29
  sentences = re.split(r'(?<=[.!?]) +', text.strip())
30
  return [' '.join(sentences[i:i+max_sentences]) for i in range(0, len(sentences), max_sentences)]
31
 
32
  def full_article_paraphrase(text):
33
  chunks = chunk_text(text)
34
+ paraphrased_chunks = []
35
+ for chunk in chunks:
36
+ if chunk.strip():
37
+ paraphrased_chunks.append(paraphrase_text(chunk.strip()))
38
+ return "\n\n".join(paraphrased_chunks)
39
 
40
  def extract_text_from_docx(file_obj):
41
  from docx import Document