Krixter commited on
Commit
fdec6bc
·
verified ·
1 Parent(s): 2794906

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +19 -4
app.py CHANGED
@@ -9,7 +9,7 @@ The app will:
9
  4. Produce one combined .txt file with: URL | Title | Description
10
  5. Zip all the saved .html files for download
11
 
12
- Built for Gradio on Hugging Face .
13
  """
14
 
15
  import os
@@ -50,8 +50,21 @@ def slugify_for_filename(url: str, index: int) -> str:
50
  return f"{index:05d}_{path}.html"
51
 
52
 
 
 
 
 
 
 
 
 
 
 
 
 
 
53
  def extract_title_and_description(html: str) -> tuple[str, str]:
54
- """Pull the best available title and description from a page's HTML."""
55
  soup = BeautifulSoup(html, "html.parser")
56
 
57
  title = ""
@@ -73,7 +86,9 @@ def extract_title_and_description(html: str) -> tuple[str, str]:
73
  # Keep everything on a single line in the combined output
74
  title = re.sub(r"\s+", " ", title)
75
  description = re.sub(r"\s+", " ", description)
76
- return title, description
 
 
77
 
78
 
79
  def fetch_one(index: int, url: str, save_html: bool, timeout: int):
@@ -186,7 +201,7 @@ def run_scraper(
186
  for res in results:
187
  f.write(f"URL: {res['url']}\n")
188
  f.write(f"Title: {res['title']}\n")
189
- f.write(f"Description: {res['description']}\n")
190
  f.write(f"Status: {res['status']}\n")
191
  f.write("-" * 60 + "\n")
192
 
 
9
  4. Produce one combined .txt file with: URL | Title | Description
10
  5. Zip all the saved .html files for download
11
 
12
+ Built for Gradio on Hugging Face Spaces.
13
  """
14
 
15
  import os
 
50
  return f"{index:05d}_{path}.html"
51
 
52
 
53
+ def extract_synopsis(description: str) -> str:
54
+ """WebNovel-style descriptions read like:
55
+ 'Read X Online for Free, written by ... and the synopsis is: <actual synopsis>'
56
+ Strip everything before the marker so only the real synopsis remains.
57
+ If the marker isn't found (non-WebNovel page), just return the description as-is.
58
+ """
59
+ marker = "the synopsis is:"
60
+ idx = description.lower().find(marker)
61
+ if idx != -1:
62
+ return description[idx + len(marker):].strip()
63
+ return description.strip()
64
+
65
+
66
  def extract_title_and_description(html: str) -> tuple[str, str]:
67
+ """Pull the best available title and synopsis/description from a page's HTML."""
68
  soup = BeautifulSoup(html, "html.parser")
69
 
70
  title = ""
 
86
  # Keep everything on a single line in the combined output
87
  title = re.sub(r"\s+", " ", title)
88
  description = re.sub(r"\s+", " ", description)
89
+
90
+ synopsis = extract_synopsis(description)
91
+ return title, synopsis
92
 
93
 
94
  def fetch_one(index: int, url: str, save_html: bool, timeout: int):
 
201
  for res in results:
202
  f.write(f"URL: {res['url']}\n")
203
  f.write(f"Title: {res['title']}\n")
204
+ f.write(f"Synopsis: {res['description']}\n")
205
  f.write(f"Status: {res['status']}\n")
206
  f.write("-" * 60 + "\n")
207