Update app.py
Browse files
app.py
CHANGED
|
@@ -3,9 +3,9 @@ import os
|
|
| 3 |
import asyncio
|
| 4 |
import json
|
| 5 |
import datetime
|
| 6 |
-
import re
|
| 7 |
-
import subprocess
|
| 8 |
-
import sys
|
| 9 |
from dotenv import load_dotenv
|
| 10 |
|
| 11 |
# --- Playwright Imports for Web Scraping ---
|
|
@@ -23,70 +23,19 @@ from langchain_core.messages import SystemMessage, HumanMessage
|
|
| 23 |
# On Hugging Face Spaces, secrets are automatically available as environment variables.
|
| 24 |
load_dotenv()
|
| 25 |
|
| 26 |
-
# --- Playwright Browser
|
| 27 |
-
# This block ensures Playwright browsers
|
| 28 |
-
#
|
| 29 |
PLAYWRIGHT_BROWSERS_PATH = os.path.expanduser("~/.cache/ms-playwright")
|
| 30 |
PLAYWRIGHT_INSTALLED_FLAG = "PLAYWRIGHT_BROWSERS_INSTALLED"
|
| 31 |
|
| 32 |
-
# Define ALL system packages required for Playwright on Debian Bookworm
|
| 33 |
-
# These will be installed manually via subprocess.
|
| 34 |
-
ALL_PLAYWRIGHT_APT_PACKAGES = [
|
| 35 |
-
"chromium", # Basic chromium package for good measure
|
| 36 |
-
"libgstreamer1.0-0",
|
| 37 |
-
"libgtk-4-1",
|
| 38 |
-
"libgraphene-1.0-0",
|
| 39 |
-
"libwoff2-1.0", # Reverted to original name, will be installed via apt-get in-app
|
| 40 |
-
"libgstreamer-plugins-base1.0-0",
|
| 41 |
-
"libgstreamer-gl1.0-0",
|
| 42 |
-
"libgstreamer-plugins-bad1.0-0",
|
| 43 |
-
"libavif15",
|
| 44 |
-
"libharfbuzz-icu0",
|
| 45 |
-
"libenchant-2-2",
|
| 46 |
-
"libhyphen0",
|
| 47 |
-
"libmanette-0.2-0",
|
| 48 |
-
"libnss3",
|
| 49 |
-
"libxss1",
|
| 50 |
-
"libasound2",
|
| 51 |
-
"libgbm-dev",
|
| 52 |
-
"libatk-bridge2.0-0",
|
| 53 |
-
"libcups2",
|
| 54 |
-
"libdrm-dev",
|
| 55 |
-
"libgdk-pixbuf2.0-0",
|
| 56 |
-
"libglib2.0-0",
|
| 57 |
-
"libjpeg-turbo8", # Reverted to original name, will be installed via apt-get in-app
|
| 58 |
-
"libnspr4",
|
| 59 |
-
"libxcb-dri3-0",
|
| 60 |
-
"libxcomposite1",
|
| 61 |
-
"libxrandr2",
|
| 62 |
-
"libxshmfence1",
|
| 63 |
-
"libxtst6",
|
| 64 |
-
"xdg-utils"
|
| 65 |
-
]
|
| 66 |
-
|
| 67 |
# Check if installation has been attempted during this container's lifetime
|
| 68 |
if not os.path.exists(os.path.join(PLAYWRIGHT_BROWSERS_PATH, "chromium")) and \
|
| 69 |
os.environ.get(PLAYWRIGHT_INSTALLED_FLAG, "false") == "false":
|
| 70 |
-
print("Playwright browsers
|
| 71 |
try:
|
| 72 |
-
#
|
| 73 |
-
|
| 74 |
-
# Use stdout=subprocess.PIPE, stderr=subprocess.PIPE to capture output and prevent it from filling up logs unless an error occurs
|
| 75 |
-
subprocess.run(["sudo", "apt-get", "update", "-qq"], check=True, capture_output=True)
|
| 76 |
-
print("Apt package lists updated.")
|
| 77 |
-
|
| 78 |
-
# Then, install all necessary system dependencies via apt-get directly
|
| 79 |
-
print(f"Installing all Playwright system dependencies via apt-get: {', '.join(ALL_PLAYWRIGHT_APT_PACKAGES)}...")
|
| 80 |
-
apt_install_command = ["sudo", "apt-get", "install", "-y"] + ALL_PLAYWRIGHT_APT_PACKAGES
|
| 81 |
-
result = subprocess.run(apt_install_command, check=True, capture_output=True, text=True)
|
| 82 |
-
print("All system dependencies installed successfully!")
|
| 83 |
-
if result.stdout:
|
| 84 |
-
print("APT Install STDOUT:\n", result.stdout)
|
| 85 |
-
if result.stderr:
|
| 86 |
-
print("APT Install STDERR:\n", result.stderr)
|
| 87 |
-
|
| 88 |
-
# Finally, run the playwright install command for the browsers themselves
|
| 89 |
-
print("Running playwright install for browsers...")
|
| 90 |
result = subprocess.run([sys.executable, "-m", "playwright", "install"], check=True, capture_output=True, text=True)
|
| 91 |
os.environ[PLAYWRIGHT_INSTALLED_FLAG] = "true" # Mark as installed for this session
|
| 92 |
print("Playwright browsers installed successfully!")
|
|
@@ -96,11 +45,11 @@ if not os.path.exists(os.path.join(PLAYWRIGHT_BROWSERS_PATH, "chromium")) and \
|
|
| 96 |
print("Playwright Install STDERR:\n", result.stderr)
|
| 97 |
|
| 98 |
except subprocess.CalledProcessError as e:
|
| 99 |
-
print(f"Error during Playwright
|
| 100 |
except Exception as e:
|
| 101 |
-
print(f"Unexpected error during Playwright browser
|
| 102 |
else:
|
| 103 |
-
print("Playwright browsers
|
| 104 |
|
| 105 |
|
| 106 |
# --- 1. API KEY CHECK ---
|
|
|
|
| 3 |
import asyncio
|
| 4 |
import json
|
| 5 |
import datetime
|
| 6 |
+
import re
|
| 7 |
+
import subprocess
|
| 8 |
+
import sys
|
| 9 |
from dotenv import load_dotenv
|
| 10 |
|
| 11 |
# --- Playwright Imports for Web Scraping ---
|
|
|
|
| 23 |
# On Hugging Face Spaces, secrets are automatically available as environment variables.
|
| 24 |
load_dotenv()
|
| 25 |
|
| 26 |
+
# --- Playwright Browser Installation Workaround for Hugging Face Spaces ---
|
| 27 |
+
# This block ensures Playwright browsers are installed when the app starts.
|
| 28 |
+
# Assumes system dependencies are handled by packages.txt during the build.
|
| 29 |
PLAYWRIGHT_BROWSERS_PATH = os.path.expanduser("~/.cache/ms-playwright")
|
| 30 |
PLAYWRIGHT_INSTALLED_FLAG = "PLAYWRIGHT_BROWSERS_INSTALLED"
|
| 31 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
# Check if installation has been attempted during this container's lifetime
|
| 33 |
if not os.path.exists(os.path.join(PLAYWRIGHT_BROWSERS_PATH, "chromium")) and \
|
| 34 |
os.environ.get(PLAYWRIGHT_INSTALLED_FLAG, "false") == "false":
|
| 35 |
+
print("Playwright browsers not found. Attempting to install them now (this might take a moment)...")
|
| 36 |
try:
|
| 37 |
+
# Run the playwright install command for the browsers themselves.
|
| 38 |
+
# This should now succeed if the system dependencies from packages.txt are met.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
result = subprocess.run([sys.executable, "-m", "playwright", "install"], check=True, capture_output=True, text=True)
|
| 40 |
os.environ[PLAYWRIGHT_INSTALLED_FLAG] = "true" # Mark as installed for this session
|
| 41 |
print("Playwright browsers installed successfully!")
|
|
|
|
| 45 |
print("Playwright Install STDERR:\n", result.stderr)
|
| 46 |
|
| 47 |
except subprocess.CalledProcessError as e:
|
| 48 |
+
print(f"Error during Playwright browser installation: {e}. STDOUT: {e.stdout.decode()} STDERR: {e.stderr.decode()}")
|
| 49 |
except Exception as e:
|
| 50 |
+
print(f"Unexpected error during Playwright browser installation: {e}")
|
| 51 |
else:
|
| 52 |
+
print("Playwright browsers already appear to be installed or installation skipped.")
|
| 53 |
|
| 54 |
|
| 55 |
# --- 1. API KEY CHECK ---
|