Spaces:
Running on Zero
Running on Zero
Commit ·
5eff2c4
1
Parent(s): 6f8d589
Fixed Playwright sandbox freezing issue in Docker
Browse files- scraper/aishe_downloader.py +4 -1
- scraper/naac_downloader.py +4 -1
- scraper/ugc_downloader.py +4 -1
scraper/aishe_downloader.py
CHANGED
|
@@ -115,7 +115,10 @@ def download_all_sections(
|
|
| 115 |
print(msg, flush=True)
|
| 116 |
|
| 117 |
with sync_playwright() as pw:
|
| 118 |
-
browser = pw.chromium.launch(
|
|
|
|
|
|
|
|
|
|
| 119 |
context = browser.new_context(accept_downloads=True)
|
| 120 |
page = context.new_page()
|
| 121 |
|
|
|
|
| 115 |
print(msg, flush=True)
|
| 116 |
|
| 117 |
with sync_playwright() as pw:
|
| 118 |
+
browser = pw.chromium.launch(
|
| 119 |
+
headless=headless,
|
| 120 |
+
args=["--no-sandbox", "--disable-dev-shm-usage"]
|
| 121 |
+
)
|
| 122 |
context = browser.new_context(accept_downloads=True)
|
| 123 |
page = context.new_page()
|
| 124 |
|
scraper/naac_downloader.py
CHANGED
|
@@ -13,7 +13,10 @@ def download_naac(output_dir: Path, headless: bool = True, log_fn=None) -> Path:
|
|
| 13 |
output_dir.mkdir(parents=True, exist_ok=True)
|
| 14 |
|
| 15 |
with sync_playwright() as p:
|
| 16 |
-
browser = p.chromium.launch(
|
|
|
|
|
|
|
|
|
|
| 17 |
context = browser.new_context(accept_downloads=True)
|
| 18 |
page = context.new_page()
|
| 19 |
|
|
|
|
| 13 |
output_dir.mkdir(parents=True, exist_ok=True)
|
| 14 |
|
| 15 |
with sync_playwright() as p:
|
| 16 |
+
browser = p.chromium.launch(
|
| 17 |
+
headless=headless,
|
| 18 |
+
args=["--no-sandbox", "--disable-dev-shm-usage"]
|
| 19 |
+
)
|
| 20 |
context = browser.new_context(accept_downloads=True)
|
| 21 |
page = context.new_page()
|
| 22 |
|
scraper/ugc_downloader.py
CHANGED
|
@@ -14,7 +14,10 @@ def download_ugc(output_dir: Path, headless: bool = True, log_fn=None) -> Path:
|
|
| 14 |
save_path = output_dir / "ugc_raw.xlsx"
|
| 15 |
|
| 16 |
with sync_playwright() as pw:
|
| 17 |
-
browser = pw.chromium.launch(
|
|
|
|
|
|
|
|
|
|
| 18 |
context = browser.new_context(accept_downloads=True)
|
| 19 |
page = context.new_page()
|
| 20 |
|
|
|
|
| 14 |
save_path = output_dir / "ugc_raw.xlsx"
|
| 15 |
|
| 16 |
with sync_playwright() as pw:
|
| 17 |
+
browser = pw.chromium.launch(
|
| 18 |
+
headless=headless,
|
| 19 |
+
args=["--no-sandbox", "--disable-dev-shm-usage"]
|
| 20 |
+
)
|
| 21 |
context = browser.new_context(accept_downloads=True)
|
| 22 |
page = context.new_page()
|
| 23 |
|