Spaces:
Sleeping
Sleeping
Pulka commited on
sync: 113 file da Baida98/AI@8b66af87 (2026-06-21 20:19 UTC)
Browse files- api/exec.py +42 -0
- tools/ast_check.py +140 -0
- tools/registry.py +8 -3
- tools/semantic_nav.py +154 -0
- tools/web_fetch.py +22 -2
api/exec.py
CHANGED
|
@@ -521,3 +521,45 @@ async def llm_fix_code(
|
|
| 521 |
|
| 522 |
return {'fixed_code': None, 'error': f'Tutti i provider falliti. Ultimo: {last_err[:300]}'} # S588: 200→300
|
| 523 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 521 |
|
| 522 |
return {'fixed_code': None, 'error': f'Tutti i provider falliti. Ultimo: {last_err[:300]}'} # S588: 200→300
|
| 523 |
|
| 524 |
+
|
| 525 |
+
# ── GAP-2: /api/exec/tool — dispatcher generico per tool calls dal frontend ────
|
| 526 |
+
# Il frontend (toolExecutor.ts) chiama questo endpoint per tool come scaffold_project
|
| 527 |
+
# che non hanno un endpoint dedicato. Il dispatcher risolve il nome tool nel TOOL_REGISTRY
|
| 528 |
+
# e invoca la funzione associata (_fn) con gli args forniti.
|
| 529 |
+
class ToolDispatchRequest(BaseModel):
|
| 530 |
+
tool: str
|
| 531 |
+
args: dict = {}
|
| 532 |
+
|
| 533 |
+
|
| 534 |
+
@router.post('/api/exec/tool')
|
| 535 |
+
async def exec_tool_dispatch(req: ToolDispatchRequest, request: Request):
|
| 536 |
+
"""GAP-2: dispatcher generico — risolve tool nel TOOL_REGISTRY e chiama _fn."""
|
| 537 |
+
_internal_token = os.getenv('INTERNAL_TOKEN', '')
|
| 538 |
+
if _internal_token and request.headers.get('X-Internal-Token') != _internal_token:
|
| 539 |
+
raise HTTPException(401, 'Unauthorized')
|
| 540 |
+
try:
|
| 541 |
+
from tools.registry import TOOL_REGISTRY # import locale — evita circular import
|
| 542 |
+
except ImportError as _ie:
|
| 543 |
+
return {'ok': False, 'error': f'TOOL_REGISTRY non disponibile: {_ie}'}
|
| 544 |
+
tool_def = TOOL_REGISTRY.get(req.tool)
|
| 545 |
+
if not tool_def:
|
| 546 |
+
available = ', '.join(list(TOOL_REGISTRY.keys())[:20])
|
| 547 |
+
return {'ok': False, 'error': f"Tool '{req.tool}' non trovato. Disponibili: {available}"}
|
| 548 |
+
_fn = tool_def.get('_fn')
|
| 549 |
+
if not _fn:
|
| 550 |
+
return {'ok': False, 'error': f"Tool '{req.tool}' non ha handler (_fn) — non eseguibile via dispatcher"}
|
| 551 |
+
try:
|
| 552 |
+
import asyncio as _asyncio
|
| 553 |
+
if _asyncio.iscoroutinefunction(_fn):
|
| 554 |
+
result = await _fn(**req.args)
|
| 555 |
+
else:
|
| 556 |
+
result = _fn(**req.args)
|
| 557 |
+
return {'ok': True, 'tool': req.tool, 'result': result}
|
| 558 |
+
except TypeError as _te:
|
| 559 |
+
# Parametri sbagliati — mostra la firma corretta
|
| 560 |
+
import inspect as _inspect
|
| 561 |
+
_sig = str(_inspect.signature(_fn))
|
| 562 |
+
return {'ok': False, 'error': f"Parametri non validi per '{req.tool}'{_sig}: {str(_te)[:200]}"}
|
| 563 |
+
except Exception as _e:
|
| 564 |
+
_logger.exception('[exec/tool] errore in %s', req.tool)
|
| 565 |
+
return {'ok': False, 'error': str(_e)[:500]}
|
tools/ast_check.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ast_check.py — P45: AST syntax check pre-esecuzione codice.
|
| 3 |
+
|
| 4 |
+
Usa ast.parse() stdlib Python (zero dipendenze) per Python.
|
| 5 |
+
Per JS/TS: esprima se disponibile, fallback balanced-bracket heuristic.
|
| 6 |
+
Integra P17-B1: sostituisce routing regex E2B per validazione sintattica.
|
| 7 |
+
|
| 8 |
+
Utilizzo:
|
| 9 |
+
from tools.ast_check import check_code_syntax
|
| 10 |
+
result = check_code_syntax(code, "python")
|
| 11 |
+
if not result.ok:
|
| 12 |
+
return f"SyntaxError: {result.error}"
|
| 13 |
+
"""
|
| 14 |
+
import ast
|
| 15 |
+
import re
|
| 16 |
+
import logging
|
| 17 |
+
from dataclasses import dataclass
|
| 18 |
+
|
| 19 |
+
_logger = logging.getLogger("tools.ast_check")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass
|
| 23 |
+
class SyntaxCheckResult:
|
| 24 |
+
ok: bool
|
| 25 |
+
error: str | None = None
|
| 26 |
+
language: str = ""
|
| 27 |
+
line: int | None = None
|
| 28 |
+
col: int | None = None
|
| 29 |
+
|
| 30 |
+
def __str__(self):
|
| 31 |
+
if self.ok:
|
| 32 |
+
return f"[AST] {self.language}: OK"
|
| 33 |
+
loc = f" (line {self.line})" if self.line else ""
|
| 34 |
+
return f"[AST] {self.language}{loc}: {self.error}"
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def check_python_syntax(code):
|
| 38 |
+
"""AST check Python con ast.parse() stdlib. Zero dipendenze."""
|
| 39 |
+
try:
|
| 40 |
+
ast.parse(code, mode="exec")
|
| 41 |
+
return SyntaxCheckResult(ok=True, language="python")
|
| 42 |
+
except SyntaxError as e:
|
| 43 |
+
return SyntaxCheckResult(
|
| 44 |
+
ok=False, language="python",
|
| 45 |
+
error=f"{type(e).__name__}: {e.msg}",
|
| 46 |
+
line=e.lineno, col=e.offset,
|
| 47 |
+
)
|
| 48 |
+
except Exception as e:
|
| 49 |
+
return SyntaxCheckResult(ok=False, language="python", error=str(e))
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _balanced_brackets(code):
|
| 53 |
+
"""Heuristica bilanciamento parentesi — fallback JS senza esprima."""
|
| 54 |
+
stack = []
|
| 55 |
+
pairs = {")": "(", "}": "{", "]": "["}
|
| 56 |
+
in_str = None
|
| 57 |
+
i = 0
|
| 58 |
+
while i < len(code):
|
| 59 |
+
ch = code[i]
|
| 60 |
+
if in_str:
|
| 61 |
+
if ch == "\\" and i + 1 < len(code):
|
| 62 |
+
i += 2
|
| 63 |
+
continue
|
| 64 |
+
if ch == in_str:
|
| 65 |
+
in_str = None
|
| 66 |
+
elif ch in ('"', "'", '`'):
|
| 67 |
+
in_str = ch
|
| 68 |
+
elif ch in "({[":
|
| 69 |
+
stack.append(ch)
|
| 70 |
+
elif ch in ")}]":
|
| 71 |
+
if not stack or stack[-1] != pairs[ch]:
|
| 72 |
+
return SyntaxCheckResult(ok=False, language="js/ts", error=f"Unmatched '{ch}' at pos {i}")
|
| 73 |
+
stack.pop()
|
| 74 |
+
i += 1
|
| 75 |
+
if stack:
|
| 76 |
+
return SyntaxCheckResult(ok=False, language="js/ts", error=f"Unclosed '{stack[-1]}'")
|
| 77 |
+
return SyntaxCheckResult(ok=True, language="js/ts")
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def check_js_syntax(code):
|
| 81 |
+
"""AST check JS/TS: esprima se disponibile, fallback balanced-bracket."""
|
| 82 |
+
try:
|
| 83 |
+
import esprima
|
| 84 |
+
esprima.parseScript(code, tolerant=False)
|
| 85 |
+
return SyntaxCheckResult(ok=True, language="js/ts")
|
| 86 |
+
except ImportError:
|
| 87 |
+
pass
|
| 88 |
+
except Exception as e:
|
| 89 |
+
msg = str(e)
|
| 90 |
+
m = re.search(r"Line (\d+)", msg)
|
| 91 |
+
line = int(m.group(1)) if m else None
|
| 92 |
+
return SyntaxCheckResult(ok=False, language="js/ts", error=msg[:200], line=line)
|
| 93 |
+
return _balanced_brackets(code)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
_LANG_MAP = {
|
| 97 |
+
"python": "python", "py": "python",
|
| 98 |
+
"javascript": "js", "js": "js", "jsx": "js",
|
| 99 |
+
"typescript": "js", "ts": "js", "tsx": "js",
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def check_code_syntax(code, language):
|
| 104 |
+
"""
|
| 105 |
+
Dispatch AST check per linguaggio. Fail-open su linguaggi non supportati.
|
| 106 |
+
Returns SyntaxCheckResult(ok, error, language, line, col).
|
| 107 |
+
"""
|
| 108 |
+
if not code or not code.strip():
|
| 109 |
+
return SyntaxCheckResult(ok=True, language=language)
|
| 110 |
+
lang = _LANG_MAP.get(language.lower().strip(), "")
|
| 111 |
+
if lang == "python":
|
| 112 |
+
result = check_python_syntax(code)
|
| 113 |
+
elif lang == "js":
|
| 114 |
+
result = check_js_syntax(code)
|
| 115 |
+
else:
|
| 116 |
+
return SyntaxCheckResult(ok=True, language=language)
|
| 117 |
+
if not result.ok:
|
| 118 |
+
_logger.warning("[AST-P45] %s syntax error: %s", language, result.error)
|
| 119 |
+
return result
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def extract_code_blocks(text):
|
| 123 |
+
"""Estrae blocchi codice da markdown. Ritorna list[(language, code)]."""
|
| 124 |
+
blocks = []
|
| 125 |
+
pattern = re.compile(r'```(\w+)?\n([\s\S]*?)```')
|
| 126 |
+
for m in pattern.finditer(text):
|
| 127 |
+
lang = (m.group(1) or "").strip()
|
| 128 |
+
code = m.group(2)
|
| 129 |
+
blocks.append((lang, code))
|
| 130 |
+
return blocks
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def check_all_code_blocks(text):
|
| 134 |
+
"""Verifica tutti i blocchi codice in markdown. Ritorna solo errori."""
|
| 135 |
+
errors = []
|
| 136 |
+
for lang, code in extract_code_blocks(text):
|
| 137 |
+
result = check_code_syntax(code, lang)
|
| 138 |
+
if not result.ok:
|
| 139 |
+
errors.append(result)
|
| 140 |
+
return errors
|
tools/registry.py
CHANGED
|
@@ -1853,7 +1853,7 @@ async def _jina_fetch(
|
|
| 1853 |
max_length=max_length,
|
| 1854 |
)
|
| 1855 |
|
| 1856 |
-
async def _python_analyze(code: str, filename: str = "<string>") -> dict:
|
| 1857 |
"""P30-B1: Analisi statica Python in-process — zero exec_engine, zero deps esterne.
|
| 1858 |
|
| 1859 |
Usa ast.parse() + visitor per:
|
|
@@ -1862,13 +1862,18 @@ async def _python_analyze(code: str, filename: str = "<string>") -> dict:
|
|
| 1862 |
- Suggerimenti actionable (funzioni troppo lunghe, nesting alto, ecc.)
|
| 1863 |
|
| 1864 |
Zero I/O, zero network. Tipicamente <5ms.
|
|
|
|
|
|
|
|
|
|
| 1865 |
"""
|
|
|
|
|
|
|
| 1866 |
import ast as _ast
|
| 1867 |
|
| 1868 |
result: dict = {"syntax_ok": False, "errors": [], "complexity": {}, "suggestions": [], "summary": ""}
|
| 1869 |
|
| 1870 |
if not code or not code.strip():
|
| 1871 |
-
result["errors"] = [{"type": "EmptyCode", "message": "Nessun codice fornito", "line": 0}]
|
| 1872 |
result["summary"] = "Codice vuoto"
|
| 1873 |
return result
|
| 1874 |
|
|
@@ -2574,7 +2579,7 @@ TOOL_REGISTRY: dict[str, dict] = {
|
|
| 2574 |
"ottenere metriche strutturali, suggerire refactoring."
|
| 2575 |
),
|
| 2576 |
"required_inputs": ["code"],
|
| 2577 |
-
"optional_inputs": {"filename": "<string>"},
|
| 2578 |
"risk_level": "low",
|
| 2579 |
"fallbacks": [],
|
| 2580 |
"_fn": _python_analyze,
|
|
|
|
| 1853 |
max_length=max_length,
|
| 1854 |
)
|
| 1855 |
|
| 1856 |
+
async def _python_analyze(code: str = "", content: str = "", filename: str = "<string>") -> dict:
|
| 1857 |
"""P30-B1: Analisi statica Python in-process — zero exec_engine, zero deps esterne.
|
| 1858 |
|
| 1859 |
Usa ast.parse() + visitor per:
|
|
|
|
| 1862 |
- Suggerimenti actionable (funzioni troppo lunghe, nesting alto, ecc.)
|
| 1863 |
|
| 1864 |
Zero I/O, zero network. Tipicamente <5ms.
|
| 1865 |
+
|
| 1866 |
+
GAP-1: accetta sia 'code' che 'content' — alias per compatibilità frontend.
|
| 1867 |
+
Il frontend (toolDefsCode.ts) invia 'content', il backend usava solo 'code'.
|
| 1868 |
"""
|
| 1869 |
+
# GAP-1: alias — frontend può inviare 'content' invece di 'code'
|
| 1870 |
+
code = code or content
|
| 1871 |
import ast as _ast
|
| 1872 |
|
| 1873 |
result: dict = {"syntax_ok": False, "errors": [], "complexity": {}, "suggestions": [], "summary": ""}
|
| 1874 |
|
| 1875 |
if not code or not code.strip():
|
| 1876 |
+
result["errors"] = [{"type": "EmptyCode", "message": "Nessun codice fornito (parametro 'code' o 'content' richiesto)", "line": 0}]
|
| 1877 |
result["summary"] = "Codice vuoto"
|
| 1878 |
return result
|
| 1879 |
|
|
|
|
| 2579 |
"ottenere metriche strutturali, suggerire refactoring."
|
| 2580 |
),
|
| 2581 |
"required_inputs": ["code"],
|
| 2582 |
+
"optional_inputs": {"filename": "<string>", "content": ""}, # GAP-1: alias per 'code' (frontend compat)
|
| 2583 |
"risk_level": "low",
|
| 2584 |
"fallbacks": [],
|
| 2585 |
"_fn": _python_analyze,
|
tools/semantic_nav.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
semantic_nav.py — Semantic Anchor Scoring (SAS)
|
| 3 |
+
|
| 4 |
+
Scoring a 3 fattori:
|
| 5 |
+
1. Keyword Match (peso 0.50) — intent semantico agente
|
| 6 |
+
2. Visual Position (peso 0.30) — struttura HTML semantica
|
| 7 |
+
3. Path Depth (peso 0.20) — profondita URL
|
| 8 |
+
|
| 9 |
+
auto_keywords_from_url(): fallback automatico se nessun intent esplicito.
|
| 10 |
+
Backward-compatible: fallback regex se beautifulsoup4 non disponibile.
|
| 11 |
+
"""
|
| 12 |
+
import re
|
| 13 |
+
import logging
|
| 14 |
+
from urllib.parse import urljoin, urlparse
|
| 15 |
+
|
| 16 |
+
_logger = logging.getLogger("tools.semantic_nav")
|
| 17 |
+
|
| 18 |
+
KEYWORD_MATCH_WEIGHT = 0.50
|
| 19 |
+
VISUAL_POSITION_WEIGHT = 0.30
|
| 20 |
+
PATH_DEPTH_WEIGHT = 0.20
|
| 21 |
+
|
| 22 |
+
VISUAL_POSITION_SCORES = {
|
| 23 |
+
"nav": 1.0,
|
| 24 |
+
"header": 0.8,
|
| 25 |
+
"main": 0.7,
|
| 26 |
+
"article": 0.6,
|
| 27 |
+
"section": 0.5,
|
| 28 |
+
"form": 0.4,
|
| 29 |
+
"aside": 0.3,
|
| 30 |
+
"footer": 0.1,
|
| 31 |
+
"default": 0.4,
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
_SKIP = re.compile(
|
| 35 |
+
r"(twitter\.com|x\.com|facebook\.com|instagram\.com|linkedin\.com|"
|
| 36 |
+
r"youtube\.com|cdn\.|analytics|pixel|tracking|googletagmanager|doubleclick|"
|
| 37 |
+
r"javascript:|mailto:|tel:|#$)",
|
| 38 |
+
re.I,
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def auto_keywords_from_url(url):
|
| 43 |
+
"""Estrae keywords dall URL come fallback intent."""
|
| 44 |
+
parsed = urlparse(url)
|
| 45 |
+
parts = []
|
| 46 |
+
host_parts = parsed.netloc.replace("www.", "").split(".")
|
| 47 |
+
parts.extend(host_parts[:-1])
|
| 48 |
+
path_parts = [s for s in parsed.path.split("/") if s and len(s) > 2]
|
| 49 |
+
parts.extend(path_parts)
|
| 50 |
+
for kv in (parsed.query or "").split("&"):
|
| 51 |
+
if "=" in kv:
|
| 52 |
+
k, v = kv.split("=", 1)
|
| 53 |
+
parts.extend([k, v])
|
| 54 |
+
_stop = {"the", "a", "an", "and", "or", "in", "on", "at", "to", "for", "of", "with"}
|
| 55 |
+
return [
|
| 56 |
+
p.lower().strip("-_")
|
| 57 |
+
for p in parts
|
| 58 |
+
if p and not p.isdigit() and p.lower() not in _stop and len(p) > 1
|
| 59 |
+
][:12]
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _get_parent_tag(tag):
|
| 63 |
+
current = tag
|
| 64 |
+
while current is not None:
|
| 65 |
+
name = getattr(current, "name", None)
|
| 66 |
+
if name in VISUAL_POSITION_SCORES:
|
| 67 |
+
return name
|
| 68 |
+
current = getattr(current, "parent", None)
|
| 69 |
+
return "default"
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _score_link(text, url, tag, keywords):
|
| 73 |
+
score = 0.0
|
| 74 |
+
text_lower = text.lower()
|
| 75 |
+
for kw in keywords:
|
| 76 |
+
if kw.lower() in text_lower:
|
| 77 |
+
score += KEYWORD_MATCH_WEIGHT * 0.6
|
| 78 |
+
break
|
| 79 |
+
parent = getattr(tag, "parent", None)
|
| 80 |
+
ctx = parent.get_text(separator=" ", strip=True)[:300] if parent else ""
|
| 81 |
+
for kw in keywords:
|
| 82 |
+
if kw.lower() in ctx.lower():
|
| 83 |
+
score += KEYWORD_MATCH_WEIGHT * 0.4
|
| 84 |
+
break
|
| 85 |
+
parent_tag = _get_parent_tag(tag)
|
| 86 |
+
score += VISUAL_POSITION_SCORES.get(parent_tag, VISUAL_POSITION_SCORES["default"]) * VISUAL_POSITION_WEIGHT
|
| 87 |
+
path_segs = [s for s in urlparse(url).path.split("/") if s]
|
| 88 |
+
score += min(len(path_segs) / 5.0, 1.0) * PATH_DEPTH_WEIGHT
|
| 89 |
+
return max(0.0, min(score, 1.0))
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _fallback_extract(html, base_url, keywords, max_links):
|
| 93 |
+
"""Regex fallback quando bs4 non disponibile."""
|
| 94 |
+
base_domain = urlparse(base_url).netloc
|
| 95 |
+
links = []
|
| 96 |
+
kw_lower = [k.lower() for k in keywords]
|
| 97 |
+
for m in re.finditer(r'<a[^>]+href=["\']([ ^"\'>#][^"\']*)["\'\'][^>]*>([^<]{2,80})</a>', html, re.I):
|
| 98 |
+
raw_href, text = m.group(1).strip(), m.group(2).strip()
|
| 99 |
+
if not raw_href or _SKIP.search(raw_href):
|
| 100 |
+
continue
|
| 101 |
+
full_url = urljoin(base_url, raw_href)
|
| 102 |
+
if not full_url.startswith(("http://", "https://")):
|
| 103 |
+
continue
|
| 104 |
+
url_lower = full_url.lower()
|
| 105 |
+
kw_hit = any(kw in text.lower() or kw in url_lower for kw in kw_lower)
|
| 106 |
+
same_domain = urlparse(full_url).netloc == base_domain
|
| 107 |
+
score = (0.5 if kw_hit else 0.2) + (0.2 if same_domain else 0.0)
|
| 108 |
+
links.append({"text": text[:60], "url": full_url, "score": round(score, 2), "context": ""})
|
| 109 |
+
links.sort(key=lambda x: x["score"], reverse=True)
|
| 110 |
+
return links[:max_links]
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def extract_and_score_links(html_content, base_url, navigation_intent_keywords=None, max_links=10):
|
| 114 |
+
"""
|
| 115 |
+
Estrae e prioritizza link con Semantic Anchor Scoring.
|
| 116 |
+
Returns: list[dict] con keys text, url, score, context. Ordinati per score.
|
| 117 |
+
"""
|
| 118 |
+
keywords = navigation_intent_keywords or auto_keywords_from_url(base_url)
|
| 119 |
+
if not keywords:
|
| 120 |
+
keywords = ["main", "content", "article"]
|
| 121 |
+
|
| 122 |
+
try:
|
| 123 |
+
from bs4 import BeautifulSoup
|
| 124 |
+
except ImportError:
|
| 125 |
+
_logger.debug("[SAS] beautifulsoup4 non disponibile — uso fallback regex")
|
| 126 |
+
return _fallback_extract(html_content, base_url, keywords, max_links)
|
| 127 |
+
|
| 128 |
+
soup = BeautifulSoup(html_content, "html.parser")
|
| 129 |
+
results = []
|
| 130 |
+
|
| 131 |
+
for a_tag in soup.find_all("a", href=True):
|
| 132 |
+
text = a_tag.get_text(strip=True)
|
| 133 |
+
raw_href = a_tag["href"]
|
| 134 |
+
if not raw_href or _SKIP.search(raw_href):
|
| 135 |
+
continue
|
| 136 |
+
full_url = urljoin(base_url, raw_href)
|
| 137 |
+
if not full_url.startswith(("http://", "https://")):
|
| 138 |
+
continue
|
| 139 |
+
score = _score_link(text, full_url, a_tag, keywords)
|
| 140 |
+
ctx = ""
|
| 141 |
+
parent = a_tag.parent
|
| 142 |
+
if parent:
|
| 143 |
+
parent_text = parent.get_text(separator=" ", strip=True)
|
| 144 |
+
idx = parent_text.find(text)
|
| 145 |
+
if idx != -1:
|
| 146 |
+
ctx = parent_text[max(0, idx - 80):idx + len(text) + 80].strip()
|
| 147 |
+
else:
|
| 148 |
+
ctx = parent_text[:160].strip()
|
| 149 |
+
results.append({"text": text[:80], "url": full_url, "score": round(score, 3), "context": ctx[:200]})
|
| 150 |
+
|
| 151 |
+
results.sort(key=lambda x: x["score"], reverse=True)
|
| 152 |
+
top = results[:max_links]
|
| 153 |
+
_logger.debug("[SAS] %s: %d scored, top %d | kw=%s", base_url, len(results), len(top), keywords[:3])
|
| 154 |
+
return top
|
tools/web_fetch.py
CHANGED
|
@@ -30,6 +30,15 @@ from typing import Optional
|
|
| 30 |
import logging
|
| 31 |
_logger = logging.getLogger("tools.web_fetch")
|
| 32 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
USER_AGENT = "Mozilla/5.0 (compatible; AgenteAI/3.0)"
|
| 34 |
MAX_CONTENT = 6000 # W-NAV: alzato da 5000 → 6000 (allineato a MAX_TEXT browser.py)
|
| 35 |
_PLAYWRIGHT_THRESHOLD = 300 # chars: se content < soglia → prova Playwright
|
|
@@ -197,7 +206,7 @@ async def _fetch_with_playwright(url: str, max_chars: int = MAX_CONTENT) -> Opti
|
|
| 197 |
|
| 198 |
# ─── Endpoint principale ──────────────────────────────────────────────────────
|
| 199 |
|
| 200 |
-
async def fetch_page(url: str, max_chars: int = MAX_CONTENT) -> dict:
|
| 201 |
"""
|
| 202 |
Fetch + estrazione testo con pipeline a 3 livelli:
|
| 203 |
L1: httpx statico + trafilatura (veloce, nessun overhead)
|
|
@@ -261,7 +270,18 @@ async def fetch_page(url: str, max_chars: int = MAX_CONTENT) -> dict:
|
|
| 261 |
result["extractor"] = "playwright"
|
| 262 |
if warning:
|
| 263 |
result["warning"] = warning
|
| 264 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
return result
|
| 266 |
|
| 267 |
except httpx.TimeoutException:
|
|
|
|
| 30 |
import logging
|
| 31 |
_logger = logging.getLogger("tools.web_fetch")
|
| 32 |
|
| 33 |
+
|
| 34 |
+
# SAS (P45-SAS): Semantic Anchor Scoring — lazy import, fail-safe fallback
|
| 35 |
+
_sas_available = False
|
| 36 |
+
try:
|
| 37 |
+
from tools import semantic_nav as _semantic_nav # noqa: PLC0415
|
| 38 |
+
_sas_available = True
|
| 39 |
+
except ImportError:
|
| 40 |
+
pass
|
| 41 |
+
|
| 42 |
USER_AGENT = "Mozilla/5.0 (compatible; AgenteAI/3.0)"
|
| 43 |
MAX_CONTENT = 6000 # W-NAV: alzato da 5000 → 6000 (allineato a MAX_TEXT browser.py)
|
| 44 |
_PLAYWRIGHT_THRESHOLD = 300 # chars: se content < soglia → prova Playwright
|
|
|
|
| 206 |
|
| 207 |
# ─── Endpoint principale ──────────────────────────────────────────────────────
|
| 208 |
|
| 209 |
+
async def fetch_page(url: str, max_chars: int = MAX_CONTENT, navigation_intent_keywords: list[str] | None = None) -> dict:
|
| 210 |
"""
|
| 211 |
Fetch + estrazione testo con pipeline a 3 livelli:
|
| 212 |
L1: httpx statico + trafilatura (veloce, nessun overhead)
|
|
|
|
| 270 |
result["extractor"] = "playwright"
|
| 271 |
if warning:
|
| 272 |
result["warning"] = warning
|
| 273 |
+
# SAS: prioritizza link per intent semantico (P45-SAS)
|
| 274 |
+
if _sas_available:
|
| 275 |
+
try:
|
| 276 |
+
result["relevant_links"] = _semantic_nav.extract_and_score_links(
|
| 277 |
+
html_content=html,
|
| 278 |
+
base_url=url,
|
| 279 |
+
navigation_intent_keywords=navigation_intent_keywords,
|
| 280 |
+
)
|
| 281 |
+
except Exception:
|
| 282 |
+
result["relevant_links"] = _extract_relevant_links(html, url)
|
| 283 |
+
else:
|
| 284 |
+
result["relevant_links"] = _extract_relevant_links(html, url)
|
| 285 |
return result
|
| 286 |
|
| 287 |
except httpx.TimeoutException:
|