Spaces:
Running
Running
File size: 6,751 Bytes
0b3ae6d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 | import sys
import os
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
try:
import requests # type: ignore
except Exception:
# Fallback lightweight wrapper around urllib when `requests` isn't available.
import urllib.request as _urllib_request
class _RequestsFallback:
class Response:
def __init__(self, text: str, status_code: int):
self.text = text
self.status_code = status_code
def raise_for_status(self):
if not (200 <= self.status_code < 300):
raise Exception(f"HTTP {self.status_code}")
def get(self, url, headers=None, timeout=None):
req = _urllib_request.Request(url, headers=headers or {})
with _urllib_request.urlopen(req, timeout=timeout) as resp:
body = resp.read().decode("utf-8", errors="ignore")
return _RequestsFallback.Response(body, resp.getcode())
requests = _RequestsFallback()
try:
from bs4 import BeautifulSoup # type: ignore
except Exception:
# Lightweight fallback when `bs4` isn't installed. Implements the
# minimal interface used in this module: constructor, calling with a
# list of tag names to remove, get_text, and find('title').
import re
import html as _html
class _FallbackElement:
def __init__(self, parent, tag):
self._parent = parent
self._tag = tag
def extract(self):
# remove all occurrences of the tag and its contents
pattern = re.compile(rf"<%s\b[^>]*>.*?</%s>" % (re.escape(self._tag), re.escape(self._tag)), re.DOTALL | re.IGNORECASE)
self._parent._html = pattern.sub("", self._parent._html)
class BeautifulSoup:
def __init__(self, html, parser=None):
self._html = html or ""
def __call__(self, tag_list):
# return a list of elements with an extract() method
return [_FallbackElement(self, tag) for tag in tag_list]
def get_text(self, separator=" ", strip=True):
# remove all tags
text = re.sub(r"<[^>]+>", "", self._html)
text = _html.unescape(text)
if strip:
text = text.strip()
if separator != " ":
# normalize whitespace according to separator
text = re.sub(r"\s+", separator, text)
else:
text = re.sub(r"\s+", " ", text)
return text
def find(self, name):
if name.lower() == 'title':
m = re.search(r"<title\b[^>]*>(.*?)</title>", self._html, re.IGNORECASE | re.DOTALL)
if m:
class _T:
def __init__(self, t):
self.text = t
return _T(_html.unescape(m.group(1).strip()))
return None
import importlib
try:
_celery_mod = importlib.import_module("celery")
Celery = getattr(_celery_mod, "Celery")
except Exception:
# Provide a lightweight fallback Celery-compatible stub so the module
# can be imported and used in environments without celery installed.
class Celery:
def __init__(self, *args, **kwargs):
self.conf = type("C", (), {})()
def task(self, *args, **kwargs):
def decorator(fn):
return fn
return decorator
try:
from dotenv import load_dotenv # type: ignore
except Exception:
# If python-dotenv isn't installed, provide a no-op fallback so
# this module can still run (env vars can come from the environment).
def load_dotenv():
return
load_dotenv()
REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0")
celery_app = Celery("vertex_tasks", broker=REDIS_URL, backend=REDIS_URL)
celery_app.conf.update(
task_serializer="json",
result_serializer="json",
accept_content=["json"],
timezone="UTC",
enable_utc=True,
worker_concurrency=4
)
@celery_app.task(name="vertex.tasks.crawl_and_index_url", bind=True, max_retries=3)
def crawl_and_index_url(self, url: str, profile: str = "public"):
print(f"๐ Worker picked up task for URL: {url} | Workspace: {profile}")
from services.ingestion import VertexIndexer
from main import nvidia_manager
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) VertexEngineCrawler/1.0"
}
try:
response = requests.get(url, headers=headers, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.text, "html.parser")
# 1. Clean the HTML
for element in soup(["script", "style", "nav", "footer", "header"]):
element.extract()
raw_text = soup.get_text(separator=" ", strip=True)[:6000]
if len(raw_text) < 150:
return {"status": "skipped", "reason": "Content body too short."}
# 2. Extract a Title purely from HTML (Not AI)
title_tag = soup.find('title')
page_title = title_tag.text.strip() if title_tag else "Untitled Webpage"
# 3. Ask AI ONLY to clean the text (No JSON formatting requests)
system_prompt = (
"You are an automated data extraction engine. I will provide raw webpage text. "
"Your ONLY job is to return the clean, readable, factual body text of the article. "
"Remove all UI elements, navigation text, and copyright notices. "
"Do NOT add any conversational text like 'Here is the text'. Output ONLY the cleaned content."
)
cleaned_text = nvidia_manager.get_completion(
model="meta/llama-3.1-8b-instruct",
messages=[
{"role": "system", "content": system_prompt},
{"role": "user", "content": f"Raw Content:\n{raw_text}"}
],
temperature=0.1
).strip()
# 4. Safely construct the dictionary in Python (No JSON Decoding needed)
if len(cleaned_text) < 50:
print(f"๐ AI rejected {url} due to low quality content.")
return {"status": "rejected", "reason": "AI determined content was not useful."}
indexer = VertexIndexer()
indexer.index_source(
title=page_title,
raw_text=cleaned_text,
source_url=url,
profile=profile
)
return {"status": "completed", "title": page_title, "profile": profile}
except Exception as exc:
print(f"Error handling crawl on {url}. Retrying... Exception: {str(exc)}")
raise self.retry(exc=exc, countdown=10) |