File size: 6,751 Bytes
0b3ae6d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
import sys
import os
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))

try:
    import requests  # type: ignore
except Exception:
    # Fallback lightweight wrapper around urllib when `requests` isn't available.
    import urllib.request as _urllib_request

    class _RequestsFallback:
        class Response:
            def __init__(self, text: str, status_code: int):
                self.text = text
                self.status_code = status_code

            def raise_for_status(self):
                if not (200 <= self.status_code < 300):
                    raise Exception(f"HTTP {self.status_code}")

        def get(self, url, headers=None, timeout=None):
            req = _urllib_request.Request(url, headers=headers or {})
            with _urllib_request.urlopen(req, timeout=timeout) as resp:
                body = resp.read().decode("utf-8", errors="ignore")
                return _RequestsFallback.Response(body, resp.getcode())

    requests = _RequestsFallback()
try:
    from bs4 import BeautifulSoup  # type: ignore
except Exception:
    # Lightweight fallback when `bs4` isn't installed. Implements the
    # minimal interface used in this module: constructor, calling with a
    # list of tag names to remove, get_text, and find('title').
    import re
    import html as _html

    class _FallbackElement:
        def __init__(self, parent, tag):
            self._parent = parent
            self._tag = tag

        def extract(self):
            # remove all occurrences of the tag and its contents
            pattern = re.compile(rf"<%s\b[^>]*>.*?</%s>" % (re.escape(self._tag), re.escape(self._tag)), re.DOTALL | re.IGNORECASE)
            self._parent._html = pattern.sub("", self._parent._html)

    class BeautifulSoup:
        def __init__(self, html, parser=None):
            self._html = html or ""

        def __call__(self, tag_list):
            # return a list of elements with an extract() method
            return [_FallbackElement(self, tag) for tag in tag_list]

        def get_text(self, separator=" ", strip=True):
            # remove all tags
            text = re.sub(r"<[^>]+>", "", self._html)
            text = _html.unescape(text)
            if strip:
                text = text.strip()
            if separator != " ":
                # normalize whitespace according to separator
                text = re.sub(r"\s+", separator, text)
            else:
                text = re.sub(r"\s+", " ", text)
            return text

        def find(self, name):
            if name.lower() == 'title':
                m = re.search(r"<title\b[^>]*>(.*?)</title>", self._html, re.IGNORECASE | re.DOTALL)
                if m:
                    class _T:
                        def __init__(self, t):
                            self.text = t
                    return _T(_html.unescape(m.group(1).strip()))
            return None
import importlib
try:
    _celery_mod = importlib.import_module("celery")
    Celery = getattr(_celery_mod, "Celery")
except Exception:
    # Provide a lightweight fallback Celery-compatible stub so the module
    # can be imported and used in environments without celery installed.
    class Celery:
        def __init__(self, *args, **kwargs):
            self.conf = type("C", (), {})()

        def task(self, *args, **kwargs):
            def decorator(fn):
                return fn
            return decorator
try:
    from dotenv import load_dotenv  # type: ignore
except Exception:
    # If python-dotenv isn't installed, provide a no-op fallback so
    # this module can still run (env vars can come from the environment).
    def load_dotenv():
        return

load_dotenv()

REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0")
celery_app = Celery("vertex_tasks", broker=REDIS_URL, backend=REDIS_URL)

celery_app.conf.update(
    task_serializer="json",
    result_serializer="json",
    accept_content=["json"],
    timezone="UTC",
    enable_utc=True,
    worker_concurrency=4
)

@celery_app.task(name="vertex.tasks.crawl_and_index_url", bind=True, max_retries=3)
def crawl_and_index_url(self, url: str, profile: str = "public"):
    print(f"๐Ÿš€ Worker picked up task for URL: {url} | Workspace: {profile}")
    
    from services.ingestion import VertexIndexer
    from main import nvidia_manager
    
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) VertexEngineCrawler/1.0"
    }
    
    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        
        soup = BeautifulSoup(response.text, "html.parser")
        
        # 1. Clean the HTML
        for element in soup(["script", "style", "nav", "footer", "header"]):
            element.extract()
            
        raw_text = soup.get_text(separator=" ", strip=True)[:6000]
        
        if len(raw_text) < 150:
            return {"status": "skipped", "reason": "Content body too short."}
            
        # 2. Extract a Title purely from HTML (Not AI)
        title_tag = soup.find('title')
        page_title = title_tag.text.strip() if title_tag else "Untitled Webpage"
        
        # 3. Ask AI ONLY to clean the text (No JSON formatting requests)
        system_prompt = (
            "You are an automated data extraction engine. I will provide raw webpage text. "
            "Your ONLY job is to return the clean, readable, factual body text of the article. "
            "Remove all UI elements, navigation text, and copyright notices. "
            "Do NOT add any conversational text like 'Here is the text'. Output ONLY the cleaned content."
        )
        
        cleaned_text = nvidia_manager.get_completion(
            model="meta/llama-3.1-8b-instruct",
            messages=[
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": f"Raw Content:\n{raw_text}"}
            ],
            temperature=0.1
        ).strip()
        
        # 4. Safely construct the dictionary in Python (No JSON Decoding needed)
        if len(cleaned_text) < 50:
            print(f"๐Ÿ›‘ AI rejected {url} due to low quality content.")
            return {"status": "rejected", "reason": "AI determined content was not useful."}
            
        indexer = VertexIndexer()
        indexer.index_source(
            title=page_title,
            raw_text=cleaned_text,
            source_url=url,
            profile=profile
        )
            
        return {"status": "completed", "title": page_title, "profile": profile}
        
    except Exception as exc:
        print(f"Error handling crawl on {url}. Retrying... Exception: {str(exc)}")
        raise self.retry(exc=exc, countdown=10)