Update app.py

app.py

@@ -22,6 +22,9 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 RESULT_COLUMNS = ["Task ID", "Question", "Submitted Answer"]
 HTTP_TIMEOUT = 45
 MAX_EXTRACTED_CHARS = 35_000
+WEBPAGE_CONNECT_TIMEOUT = 8
+WEBPAGE_READ_TIMEOUT = 20
+MAX_WEBPAGE_CHARS = 8_000
 DEFAULT_MAIN_MODEL = "groq/qwen/qwen3.6-27b"
 DEFAULT_GROQ_REVIEW_MODEL = "groq/qwen/qwen3.6-27b"
 TASK_FILE_CACHE = {}
@@ -248,8 +251,9 @@ class OpenWebPageTool(Tool):
     description = (
         "Opens and reads one exact HTTP/HTTPS page. Use it after web_search to "
         "verify article text, tables, archives, papers, and linked sources. "
-        "It automatically retries blocked HTML pages through a text mirror. "
-        "It is not a binary-file downloader."
+        "It returns concise main-page content while preserving links near the "
+        "end, and retries blocked HTML through a text mirror. It is not a "
+        "binary-file downloader."
     )
     inputs = {
         "url": {
@@ -283,7 +287,7 @@ class OpenWebPageTool(Tool):
                 response = requests.get(
                     target,
                     headers=headers,
-                    timeout=HTTP_TIMEOUT,
+                    timeout=(WEBPAGE_CONNECT_TIMEOUT, WEBPAGE_READ_TIMEOUT),
                     allow_redirects=True,
                 )
                 response.raise_for_status()
@@ -300,10 +304,68 @@ class OpenWebPageTool(Tool):
 
                 text = response.text
                 if "html" in content_type:
-                    text = markdownify(text)
-                text = text.strip()
-                if len(text) > 30_000:
-                    text = text[:30_000] + "\n[page truncated]"
+                    from bs4 import BeautifulSoup
+                    from trafilatura import extract
+                    from urllib.parse import urljoin
+
+                    soup = BeautifulSoup(text, "html.parser")
+                    for element in soup.select(
+                        "script, style, noscript, nav, header, footer, form, "
+                        "aside, iframe"
+                    ):
+                        element.decompose()
+                    main_content = (
+                        soup.find("article")
+                        or soup.find("main")
+                        or soup.body
+                        or soup
+                    )
+                    extracted = extract(
+                        text,
+                        url=target,
+                        output_format="markdown",
+                        include_comments=False,
+                        include_tables=True,
+                        include_links=True,
+                        favor_recall=True,
+                    )
+                    text = (
+                        extracted
+                        if extracted and len(extracted.strip()) >= 200
+                        else markdownify(str(main_content))
+                    )
+
+                    # Article extractors sometimes classify bibliography links
+                    # as navigation. Append a compact link index so the agent
+                    # can still open papers and primary sources cited at the end.
+                    source_links = []
+                    seen_links = set()
+                    for anchor in main_content.find_all("a", href=True):
+                        label = " ".join(anchor.get_text(" ", strip=True).split())
+                        absolute_url = urljoin(target, anchor["href"])
+                        if (
+                            label
+                            and absolute_url.startswith(("http://", "https://"))
+                            and absolute_url not in seen_links
+                        ):
+                            seen_links.add(absolute_url)
+                            source_links.append(
+                                f"- [{label[:160]}]({absolute_url})"
+                            )
+                    if source_links:
+                        text += (
+                            "\n\nSource links found on the page:\n"
+                            + "\n".join(source_links[-25:])
+                        )
+                text = re.sub(r"\n{3,}", "\n\n", text).strip()
+                if len(text) > MAX_WEBPAGE_CHARS:
+                    head_size = 5_000
+                    tail_size = MAX_WEBPAGE_CHARS - head_size
+                    text = (
+                        text[:head_size]
+                        + "\n\n[page middle omitted to conserve tokens]\n\n"
+                        + text[-tail_size:]
+                    )
                 return text or "The page was retrieved but contained no text."
             except Exception as exc:
                 errors.append(f"{target}: {exc}")
@@ -489,7 +551,8 @@ class BasicAgent:
             model_id=model_id,
             api_key=groq_api_key,
             temperature=0,
-            max_tokens=2_000,
+            max_tokens=1_200,
+            requests_per_minute=2,
         )
         self.hf_token = hf_token
         self.model_id = model_id
@@ -721,6 +784,16 @@ include reasoning, explanations, labels, Markdown, citations, or the words
                 except ValueError:
                     pass
 
+        if "award number" in question_lower or "grant number" in question_lower:
+            identifiers = re.findall(r"\b[A-Z0-9][A-Z0-9-]{5,}\b", text.upper())
+            identifiers = [
+                value
+                for value in identifiers
+                if re.search(r"[A-Z]", value) and re.search(r"\d", value)
+            ]
+            if identifiers:
+                return identifiers[-1].strip(" .,:;\"'")
+
         person_question = (
             question_lower.startswith("who ")
             or " who " in f" {question_lower} "