Spaces:
Sleeping
Sleeping
Update web_scraper_tool.py
Browse files- web_scraper_tool.py +146 -251
web_scraper_tool.py
CHANGED
|
@@ -1,33 +1,11 @@
|
|
| 1 |
# -*- coding: utf-8 -*-
|
| 2 |
import requests
|
| 3 |
-
from bs4 import BeautifulSoup
|
| 4 |
-
from fpdf import FPDF
|
| 5 |
-
from urllib.parse import urlparse, urlunparse
|
| 6 |
import tempfile
|
| 7 |
import os
|
| 8 |
-
import re
|
| 9 |
-
import traceback
|
| 10 |
-
|
| 11 |
-
def clean_problematic_chars(text, use_unicode_font=False):
|
| 12 |
-
# ... (esta función permanece igual que en la versión anterior)
|
| 13 |
-
if use_unicode_font:
|
| 14 |
-
text = text.replace('\u00A0', ' ')
|
| 15 |
-
else:
|
| 16 |
-
replacements = {
|
| 17 |
-
'\u20AC': 'EUR', '\u00A3': 'GBP', '\u00A5': 'JPY', '\u2013': '-',
|
| 18 |
-
'\u2014': '--', '\u2018': "'", '\u2019': "'", '\u201C': '"',
|
| 19 |
-
'\u201D': '"', '\u2026': '...', '\u00A0': ' ', '\u00A9': '(C)',
|
| 20 |
-
'\u00AE': '(R)',
|
| 21 |
-
}
|
| 22 |
-
for problematic, replacement in replacements.items():
|
| 23 |
-
text = text.replace(problematic, replacement)
|
| 24 |
-
|
| 25 |
-
text = "".join(c for c in text if c.isprintable() or c in ('\n', '\r', '\t'))
|
| 26 |
-
try:
|
| 27 |
-
text = text.encode('latin-1', 'ignore').decode('latin-1')
|
| 28 |
-
except Exception:
|
| 29 |
-
text = "".join(c for c in text if ord(c) < 256 or c in ('\n', '\r', '\t'))
|
| 30 |
-
return "".join(c for c in text if c.isprintable() or c in ('\n', '\r', '\t'))
|
| 31 |
|
| 32 |
class WebScrapperTool:
|
| 33 |
def __init__(self):
|
|
@@ -35,92 +13,30 @@ class WebScrapperTool:
|
|
| 35 |
self.session.headers.update({
|
| 36 |
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
| 37 |
})
|
| 38 |
-
self.font_path
|
| 39 |
-
self.using_unicode_font = False # Se establecerá después de intentar añadir la fuente
|
| 40 |
-
|
| 41 |
-
# El registro de la fuente se hará una vez por instancia de PDF, no globalmente aquí.
|
| 42 |
if not self.font_path:
|
| 43 |
-
print("
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
#
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
#
|
| 56 |
-
|
| 57 |
-
if os.path.exists(
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
# Buscar en una subcarpeta 'fonts' relativa al script
|
| 61 |
-
path2 = os.path.join(script_dir, 'fonts', font_file_name)
|
| 62 |
-
if os.path.exists(path2):
|
| 63 |
-
return os.path.abspath(path2), font_family_name_in_fpdf
|
| 64 |
-
|
| 65 |
-
# Fallback si no se encuentra
|
| 66 |
-
return None, 'Arial'
|
| 67 |
-
|
| 68 |
-
def _setup_pdf_font(self, pdf_instance):
|
| 69 |
-
"""Intenta añadir la fuente Unicode al objeto PDF y establece el estado."""
|
| 70 |
-
current_font_to_use = 'Arial' # Por defecto
|
| 71 |
-
self.using_unicode_font = False
|
| 72 |
-
|
| 73 |
-
if self.font_path: # Si encontramos el archivo .ttf
|
| 74 |
-
try:
|
| 75 |
-
# Solo registramos el estilo regular. FPDF no "crea" bold/italic de un solo .ttf
|
| 76 |
-
pdf_instance.add_font(self.font_family_for_fpdf, '', self.font_path, uni=True)
|
| 77 |
-
# También registrar alias para Bold, Italic, BoldItalic si tuviéramos los archivos .ttf correspondientes.
|
| 78 |
-
# Como no los tenemos para DejaVuSansCondensed, no podemos usar 'B', 'I' con esta familia.
|
| 79 |
-
# pdf_instance.add_font(self.font_family_for_fpdf, 'B', "DejaVuSansCondensed-Bold.ttf", uni=True) # EJEMPLO si tuvieras el archivo
|
| 80 |
-
|
| 81 |
-
current_font_to_use = self.font_family_for_fpdf
|
| 82 |
-
self.using_unicode_font = True
|
| 83 |
-
print(f"INFO: Fuente Unicode '{self.font_family_for_fpdf}' (regular) registrada en FPDF.")
|
| 84 |
-
except Exception as e_font:
|
| 85 |
-
print(f"ERROR al registrar fuente Unicode '{self.font_family_for_fpdf}' desde '{self.font_path}': {e_font}")
|
| 86 |
-
traceback.print_exc()
|
| 87 |
-
print("ADVERTENCIA: Recurriendo a fuente Arial debido a error con fuente Unicode.")
|
| 88 |
-
# self.using_unicode_font ya es False
|
| 89 |
-
else:
|
| 90 |
-
print("INFO: No se encontró archivo de fuente DejaVu. Usando Arial (soporte Unicode limitado).")
|
| 91 |
-
# self.using_unicode_font ya es False
|
| 92 |
-
|
| 93 |
-
return current_font_to_use
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
def _set_font_with_style(self, pdf_instance, family, style, size):
|
| 97 |
-
"""Wrapper para set_font que maneja si podemos usar estilos con la fuente actual."""
|
| 98 |
-
if family == self.font_family_for_fpdf and self.using_unicode_font:
|
| 99 |
-
# Si es nuestra fuente DejaVu y es Unicode, FPDF no puede aplicar 'B' o 'I'
|
| 100 |
-
# a menos que hayamos registrado explícitamente las variantes Bold/Italic de la fuente.
|
| 101 |
-
# Como solo registramos la regular, ignoramos el estilo para DejaVu.
|
| 102 |
-
# La "negrita" se simulará con subrayado o se omitirá.
|
| 103 |
-
if style == 'B':
|
| 104 |
-
# Podríamos intentar pdf.set_text_shaping(True) y luego usar HTML con <b> o <strong>
|
| 105 |
-
# pero es complejo. O FPDF tiene un render_mode para pseudo-bold.
|
| 106 |
-
# Por ahora, simplemente la usamos regular. O, para simular:
|
| 107 |
-
# pdf_instance.set_draw_color(0) # Asegurar color de texto
|
| 108 |
-
# pdf_instance.set_line_width(0.2) # Ancho de línea para "negrita"
|
| 109 |
-
# pdf_instance.text_mode = 2 # Fill, then stroke
|
| 110 |
-
pdf_instance.set_font(family, '', size) # Usar estilo regular
|
| 111 |
-
# pdf_instance.cell(..., ln=3) # ln=3 para subrayar si el texto no es multilínea
|
| 112 |
-
elif style == 'I':
|
| 113 |
-
pdf_instance.set_font(family, '', size) # Usar estilo regular, FPDF no simula itálica para TTF unicode fácilmente
|
| 114 |
-
else: # Estilo regular o vacío
|
| 115 |
-
pdf_instance.set_font(family, '', size)
|
| 116 |
-
else: # Para fuentes core como Arial, FPDF maneja 'B', 'I' internamente
|
| 117 |
-
pdf_instance.set_font(family, style, size)
|
| 118 |
-
|
| 119 |
|
| 120 |
def normalize_url(self, url: str) -> str:
|
| 121 |
-
# ... (sin cambios)
|
| 122 |
url = url.strip()
|
| 123 |
parsed_url = urlparse(url)
|
|
|
|
| 124 |
scheme = parsed_url.scheme
|
| 125 |
if not scheme:
|
| 126 |
if not parsed_url.netloc and parsed_url.path:
|
|
@@ -131,54 +47,56 @@ class WebScrapperTool:
|
|
| 131 |
new_path = '/'.join(path_parts[1:])
|
| 132 |
parsed_url = parsed_url._replace(scheme="https", netloc=new_netloc, path=new_path)
|
| 133 |
else:
|
| 134 |
-
parsed_url = parsed_url._replace(scheme="https", path=parsed_url.path)
|
| 135 |
-
else:
|
| 136 |
parsed_url = parsed_url._replace(scheme="https")
|
|
|
|
| 137 |
return urlunparse(parsed_url)
|
| 138 |
|
| 139 |
def is_image_url(self, url: str) -> bool:
|
| 140 |
-
# ... (sin cambios)
|
| 141 |
image_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp']
|
| 142 |
parsed_url = urlparse(url)
|
| 143 |
return any(parsed_url.path.lower().endswith(ext) for ext in image_extensions)
|
| 144 |
|
| 145 |
-
def _get_content(self, url: str
|
| 146 |
-
# ... (sin cambios)
|
| 147 |
try:
|
| 148 |
-
|
| 149 |
-
response = self.session.get(url, timeout=20, allow_redirects=True, stream=stream_setting)
|
| 150 |
response.raise_for_status()
|
|
|
|
| 151 |
content_type_header = response.headers.get('content-type', '').lower()
|
| 152 |
|
| 153 |
-
if 'image' in content_type_header or
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
return None, response.content, content_type_header
|
| 158 |
|
|
|
|
| 159 |
try:
|
| 160 |
content_text = response.content.decode('utf-8')
|
| 161 |
except UnicodeDecodeError:
|
| 162 |
-
content_text = response.text
|
|
|
|
| 163 |
return content_text, response.content, content_type_header
|
| 164 |
except requests.exceptions.Timeout:
|
| 165 |
-
return None, None, f"Error: Timeout al acceder a la URL: {url}"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
except requests.exceptions.RequestException as e:
|
| 167 |
-
return None, None, f"Error de conexión/HTTP
|
| 168 |
-
|
| 169 |
|
| 170 |
def scrape_to_text(self, url: str):
|
| 171 |
-
# ... (sin cambios)
|
| 172 |
text_content, _, content_type_or_error_msg = self._get_content(url)
|
| 173 |
|
| 174 |
-
if text_content is None and not ('image' in content_type_or_error_msg):
|
| 175 |
if isinstance(content_type_or_error_msg, str) and content_type_or_error_msg.startswith("Error:"):
|
| 176 |
return {'status': 'error', 'message': content_type_or_error_msg, 'url': url}
|
| 177 |
|
| 178 |
final_text = ""
|
| 179 |
-
if 'text/html' in content_type_or_error_msg
|
| 180 |
soup = BeautifulSoup(text_content, 'html.parser')
|
| 181 |
-
for element in soup(["script", "style", "nav", "footer", "aside", "form", "button", "input", "header"
|
| 182 |
element.decompose()
|
| 183 |
body = soup.find('body')
|
| 184 |
if body:
|
|
@@ -186,13 +104,14 @@ class WebScrapperTool:
|
|
| 186 |
final_text = "\n".join(text_items)
|
| 187 |
else:
|
| 188 |
final_text = "\n".join([s.strip() for s in soup.stripped_strings if s.strip()])
|
|
|
|
| 189 |
elif 'text/plain' in content_type_or_error_msg and text_content:
|
| 190 |
final_text = text_content
|
| 191 |
elif self.is_image_url(url) or ('image' in content_type_or_error_msg):
|
| 192 |
-
return {'status': 'error', 'message': f"La URL apunta a una imagen. El formato TXT es para contenido textual.", 'url': url}
|
| 193 |
-
elif text_content:
|
| 194 |
final_text = text_content
|
| 195 |
-
else:
|
| 196 |
error_message = content_type_or_error_msg if isinstance(content_type_or_error_msg, str) else f"Tipo de contenido no soportado para TXT: {content_type_or_error_msg}"
|
| 197 |
return {'status': 'error', 'message': error_message, 'url': url}
|
| 198 |
|
|
@@ -207,143 +126,119 @@ class WebScrapperTool:
|
|
| 207 |
except Exception as e:
|
| 208 |
return {'status': 'error', 'message': f"Error al escribir archivo TXT: {str(e)}", 'url': url}
|
| 209 |
|
| 210 |
-
|
| 211 |
def scrape_to_pdf(self, url: str):
|
| 212 |
-
|
| 213 |
-
text_content, raw_content, content_type_or_error_msg = self._get_content(url)
|
| 214 |
|
| 215 |
-
|
| 216 |
-
|
| 217 |
|
| 218 |
-
|
| 219 |
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
# ... (lógica de imagen directa, sin cambios aquí)
|
| 229 |
try:
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
img_path = tmp_img.name
|
| 240 |
-
|
| 241 |
-
try:
|
| 242 |
-
page_width = pdf.w - 2 * pdf.l_margin
|
| 243 |
-
pdf.image(img_path, x=pdf.l_margin, y=pdf.t_margin, w=page_width)
|
| 244 |
-
except RuntimeError as re_img:
|
| 245 |
-
return {'status': 'error', 'message': f"Error al añadir imagen directa al PDF ({img_suffix}): {str(re_img)}", 'url': url}
|
| 246 |
-
finally:
|
| 247 |
-
if os.path.exists(img_path): os.unlink(img_path)
|
| 248 |
-
except Exception as e_img:
|
| 249 |
-
return {'status': 'error', 'message': f"Error procesando imagen directa para PDF: {str(e_img)}", 'url': url}
|
| 250 |
-
|
| 251 |
-
elif 'text/html' in content_type_or_error_msg and text_content:
|
| 252 |
-
soup = BeautifulSoup(text_content, 'html.parser')
|
| 253 |
-
|
| 254 |
-
self._set_font_with_style(pdf, active_font_family, 'B', 12)
|
| 255 |
-
cleaned_url_title = clean_problematic_chars(f"Contenido de: {url}", self.using_unicode_font)
|
| 256 |
-
pdf.multi_cell(0, 8, cleaned_url_title)
|
| 257 |
-
pdf.ln(6)
|
| 258 |
-
self._set_font_with_style(pdf, active_font_family, '', 11) # Reset a normal
|
| 259 |
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
img_url_abs = urljoin(url, img_src)
|
| 274 |
-
pdf.ln(5)
|
| 275 |
-
try:
|
| 276 |
-
_, img_data, img_content_type = self._get_content(img_url_abs, is_for_image_download=True)
|
| 277 |
-
if img_data and 'image' in img_content_type:
|
| 278 |
-
img_sfx = '.' + img_content_type.split('/')[-1].split(';')[0].strip();
|
| 279 |
-
if img_sfx == '.': img_sfx = '.jpg'
|
| 280 |
-
with tempfile.NamedTemporaryFile(delete=False, suffix=img_sfx) as tmp_img_file:
|
| 281 |
-
tmp_img_file.write(img_data); tmp_img_path = tmp_img_file.name
|
| 282 |
-
try:
|
| 283 |
-
page_w = pdf.w - 2 * pdf.l_margin
|
| 284 |
-
pdf.image(tmp_img_path, x=None, y=None, w=page_w); pdf.ln(2)
|
| 285 |
-
except RuntimeError as e_fpdf_img:
|
| 286 |
-
print(f"Error FPDF al añadir imagen {img_url_abs}: {e_fpdf_img}")
|
| 287 |
-
self._set_font_with_style(pdf, active_font_family, 'I', 9)
|
| 288 |
-
err_img_msg = clean_problematic_chars(f"[Error render img: {img_url_abs} - {e_fpdf_img}]", self.using_unicode_font)
|
| 289 |
-
pdf.multi_cell(0,5, err_img_msg)
|
| 290 |
-
self._set_font_with_style(pdf, active_font_family, '', 11)
|
| 291 |
-
finally:
|
| 292 |
-
if os.path.exists(tmp_img_path): os.unlink(tmp_img_path)
|
| 293 |
-
except Exception as e_dl_img:
|
| 294 |
-
print(f"Excepción al descargar/procesar imagen {img_url_abs}: {e_dl_img}")
|
| 295 |
-
self._set_font_with_style(pdf, active_font_family, 'I', 9)
|
| 296 |
-
err_dl_msg = clean_problematic_chars(f"[Error download img: {img_url_abs}]", self.using_unicode_font)
|
| 297 |
-
pdf.multi_cell(0,5, err_dl_msg)
|
| 298 |
-
self._set_font_with_style(pdf, active_font_family, '', 11)
|
| 299 |
-
pdf.ln(5)
|
| 300 |
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
current_style = ''
|
| 311 |
-
font_size = 11
|
| 312 |
-
if element.name.startswith('h') and len(element.name) == 2:
|
| 313 |
-
try:
|
| 314 |
-
header_level = int(element.name[1])
|
| 315 |
-
font_size = max(8, 16 - header_level) # h1=15, h2=14 ... h6=10
|
| 316 |
-
current_style = 'B' # Solicitar negrita
|
| 317 |
-
except ValueError: pass # Usar defaults
|
| 318 |
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
|
|
|
| 326 |
|
| 327 |
-
elif 'text/plain' in content_type_or_error_msg and text_content:
|
| 328 |
-
self._set_font_with_style(pdf, active_font_family, 'B', 12)
|
| 329 |
-
cleaned_url_title = clean_problematic_chars(f"Contenido de: {url}", self.using_unicode_font)
|
| 330 |
-
pdf.multi_cell(0, 8, cleaned_url_title)
|
| 331 |
-
pdf.ln(6)
|
| 332 |
-
self._set_font_with_style(pdf, active_font_family, '', 11)
|
| 333 |
-
clean_text_content = clean_problematic_chars(text_content, self.using_unicode_font)
|
| 334 |
-
pdf.multi_cell(0, 7, clean_text_content)
|
| 335 |
-
else:
|
| 336 |
-
return {'status': 'error', 'message': f"Tipo de contenido no soportado o vacío para PDF: {content_type_or_error_msg}", 'url': url}
|
| 337 |
|
| 338 |
with tempfile.NamedTemporaryFile(delete=False, mode='wb', suffix='.pdf') as tmp_file:
|
| 339 |
-
pdf_output_bytes = pdf.output(dest='S')
|
| 340 |
tmp_file.write(pdf_output_bytes)
|
| 341 |
filepath = tmp_file.name
|
| 342 |
return {'status': 'success', 'file': filepath, 'url': url}
|
| 343 |
-
|
| 344 |
-
|
| 345 |
tb_str = traceback.format_exc()
|
| 346 |
-
error_message = f"Error al generar PDF: {str(
|
| 347 |
-
if len(error_message) >
|
| 348 |
-
print(f"ERROR CRÍTICO en scrape_to_pdf: {error_message}")
|
| 349 |
return {'status': 'error', 'message': error_message, 'url': url}
|
|
|
|
| 1 |
# -*- coding: utf-8 -*-
|
| 2 |
import requests
|
| 3 |
+
from bs4 import BeautifulSoup
|
| 4 |
+
from fpdf import FPDF # Usaremos fpdf2, que se importa así
|
| 5 |
+
from urllib.parse import urlparse, urlunparse
|
| 6 |
import tempfile
|
| 7 |
import os
|
| 8 |
+
import re # Para expresiones regulares
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
class WebScrapperTool:
|
| 11 |
def __init__(self):
|
|
|
|
| 13 |
self.session.headers.update({
|
| 14 |
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
| 15 |
})
|
| 16 |
+
self.font_path = self._find_font()
|
|
|
|
|
|
|
|
|
|
| 17 |
if not self.font_path:
|
| 18 |
+
print("Advertencia: No se encontró 'DejaVuSansCondensed.ttf'. Se usará Arial para PDFs (soporte Unicode limitado).")
|
| 19 |
+
print("Para mejor soporte Unicode, descarga DejaVuSansCondensed.ttf y colócalo en el directorio del script o en una subcarpeta 'fonts'.")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _find_font(self):
|
| 23 |
+
font_name = 'DejaVuSansCondensed.ttf'
|
| 24 |
+
# Comprobar en el directorio actual
|
| 25 |
+
if os.path.exists(font_name):
|
| 26 |
+
return font_name
|
| 27 |
+
# Comprobar en un subdirectorio 'fonts'
|
| 28 |
+
if os.path.exists(os.path.join('fonts', font_name)):
|
| 29 |
+
return os.path.join('fonts', font_name)
|
| 30 |
+
# Si tienes una ruta absoluta o específica en tu entorno de despliegue, puedes añadirla aquí
|
| 31 |
+
# Ejemplo para Hugging Face Spaces si subes la fuente a una carpeta 'assets':
|
| 32 |
+
# if os.path.exists(os.path.join('assets', font_name)):
|
| 33 |
+
# return os.path.join('assets', font_name)
|
| 34 |
+
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
def normalize_url(self, url: str) -> str:
|
|
|
|
| 37 |
url = url.strip()
|
| 38 |
parsed_url = urlparse(url)
|
| 39 |
+
|
| 40 |
scheme = parsed_url.scheme
|
| 41 |
if not scheme:
|
| 42 |
if not parsed_url.netloc and parsed_url.path:
|
|
|
|
| 47 |
new_path = '/'.join(path_parts[1:])
|
| 48 |
parsed_url = parsed_url._replace(scheme="https", netloc=new_netloc, path=new_path)
|
| 49 |
else:
|
| 50 |
+
parsed_url = parsed_url._replace(scheme="https", path=parsed_url.path) # Mantener path si no parece dominio
|
| 51 |
+
else: # Netloc existe o ambos están vacíos
|
| 52 |
parsed_url = parsed_url._replace(scheme="https")
|
| 53 |
+
|
| 54 |
return urlunparse(parsed_url)
|
| 55 |
|
| 56 |
def is_image_url(self, url: str) -> bool:
|
|
|
|
| 57 |
image_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp']
|
| 58 |
parsed_url = urlparse(url)
|
| 59 |
return any(parsed_url.path.lower().endswith(ext) for ext in image_extensions)
|
| 60 |
|
| 61 |
+
def _get_content(self, url: str):
|
|
|
|
| 62 |
try:
|
| 63 |
+
response = self.session.get(url, timeout=20, allow_redirects=True, stream=True if self.is_image_url(url) else False)
|
|
|
|
| 64 |
response.raise_for_status()
|
| 65 |
+
|
| 66 |
content_type_header = response.headers.get('content-type', '').lower()
|
| 67 |
|
| 68 |
+
if 'image' in content_type_header or self.is_image_url(url): # Manejo especial para imágenes
|
| 69 |
+
# Para imágenes, queremos el contenido binario crudo
|
| 70 |
+
raw_content = response.content # Leer todo el contenido de la imagen
|
| 71 |
+
return None, raw_content, content_type_header # text_content es None
|
|
|
|
| 72 |
|
| 73 |
+
# Para contenido textual
|
| 74 |
try:
|
| 75 |
content_text = response.content.decode('utf-8')
|
| 76 |
except UnicodeDecodeError:
|
| 77 |
+
content_text = response.text # Fallback a la detección de encoding de requests
|
| 78 |
+
|
| 79 |
return content_text, response.content, content_type_header
|
| 80 |
except requests.exceptions.Timeout:
|
| 81 |
+
return None, None, f"Error: Timeout al intentar acceder a la URL: {url}"
|
| 82 |
+
except requests.exceptions.TooManyRedirects:
|
| 83 |
+
return None, None, f"Error: Demasiados redirects para la URL: {url}"
|
| 84 |
+
except requests.exceptions.SSLError:
|
| 85 |
+
return None, None, f"Error: Problema de SSL con la URL: {url}. Intenta con http:// o verifica el certificado."
|
| 86 |
except requests.exceptions.RequestException as e:
|
| 87 |
+
return None, None, f"Error de conexión/HTTP: {str(e)}"
|
|
|
|
| 88 |
|
| 89 |
def scrape_to_text(self, url: str):
|
|
|
|
| 90 |
text_content, _, content_type_or_error_msg = self._get_content(url)
|
| 91 |
|
| 92 |
+
if text_content is None and not ('image' in content_type_or_error_msg): # Si es un error real, no una imagen
|
| 93 |
if isinstance(content_type_or_error_msg, str) and content_type_or_error_msg.startswith("Error:"):
|
| 94 |
return {'status': 'error', 'message': content_type_or_error_msg, 'url': url}
|
| 95 |
|
| 96 |
final_text = ""
|
| 97 |
+
if 'text/html' in content_type_or_error_msg:
|
| 98 |
soup = BeautifulSoup(text_content, 'html.parser')
|
| 99 |
+
for element in soup(["script", "style", "nav", "footer", "aside", "form", "button", "input", "header"]):
|
| 100 |
element.decompose()
|
| 101 |
body = soup.find('body')
|
| 102 |
if body:
|
|
|
|
| 104 |
final_text = "\n".join(text_items)
|
| 105 |
else:
|
| 106 |
final_text = "\n".join([s.strip() for s in soup.stripped_strings if s.strip()])
|
| 107 |
+
|
| 108 |
elif 'text/plain' in content_type_or_error_msg and text_content:
|
| 109 |
final_text = text_content
|
| 110 |
elif self.is_image_url(url) or ('image' in content_type_or_error_msg):
|
| 111 |
+
return {'status': 'error', 'message': f"La URL apunta a una imagen. El formato TXT es para contenido textual. Intente el formato PDF para imágenes.", 'url': url}
|
| 112 |
+
elif text_content: # Otro tipo de contenido decodificado como texto
|
| 113 |
final_text = text_content
|
| 114 |
+
else: # Error o tipo no manejado
|
| 115 |
error_message = content_type_or_error_msg if isinstance(content_type_or_error_msg, str) else f"Tipo de contenido no soportado para TXT: {content_type_or_error_msg}"
|
| 116 |
return {'status': 'error', 'message': error_message, 'url': url}
|
| 117 |
|
|
|
|
| 126 |
except Exception as e:
|
| 127 |
return {'status': 'error', 'message': f"Error al escribir archivo TXT: {str(e)}", 'url': url}
|
| 128 |
|
|
|
|
| 129 |
def scrape_to_pdf(self, url: str):
|
| 130 |
+
text_content, raw_content, content_type_or_error_msg = self._get_content(url)
|
|
|
|
| 131 |
|
| 132 |
+
if text_content is None and raw_content is None: # Error al obtener contenido
|
| 133 |
+
return {'status': 'error', 'message': content_type_or_error_msg, 'url': url}
|
| 134 |
|
| 135 |
+
is_likely_image = 'image' in content_type_or_error_msg or self.is_image_url(url)
|
| 136 |
|
| 137 |
+
if is_likely_image and raw_content:
|
| 138 |
+
try:
|
| 139 |
+
pdf = FPDF()
|
| 140 |
+
pdf.add_page()
|
| 141 |
+
|
| 142 |
+
img_suffix = '.' + content_type_or_error_msg.split('/')[-1].split(';')[0] # ej: .jpeg, .png
|
| 143 |
+
if img_suffix == '.': img_suffix = '.jpg' # Fallback
|
| 144 |
|
| 145 |
+
with tempfile.NamedTemporaryFile(delete=False, suffix=img_suffix) as tmp_img:
|
| 146 |
+
tmp_img.write(raw_content)
|
| 147 |
+
img_path = tmp_img.name
|
| 148 |
+
|
|
|
|
| 149 |
try:
|
| 150 |
+
page_width = pdf.w - 2 * pdf.l_margin
|
| 151 |
+
# Intentar obtener dimensiones de la imagen para ajustar si es muy grande
|
| 152 |
+
# Esto requiere Pillow, que no hemos añadido como dependencia para mantenerlo simple.
|
| 153 |
+
# Por ahora, solo la ajustamos al ancho de página.
|
| 154 |
+
pdf.image(img_path, x=pdf.l_margin, y=pdf.t_margin, w=page_width)
|
| 155 |
+
except RuntimeError as re_img:
|
| 156 |
+
os.unlink(img_path)
|
| 157 |
+
return {'status': 'error', 'message': f"Error al añadir imagen al PDF (formato {img_suffix} podría no ser compatible con FPDF o imagen corrupta): {str(re_img)}", 'url': url}
|
| 158 |
+
finally:
|
| 159 |
+
if os.path.exists(img_path): # Asegurarse de que exista antes de borrar
|
| 160 |
+
os.unlink(img_path)
|
| 161 |
+
|
| 162 |
+
with tempfile.NamedTemporaryFile(delete=False, mode='wb', suffix='.pdf') as tmp_file:
|
| 163 |
+
pdf_bytes = pdf.output(dest='S').encode('latin-1')
|
| 164 |
+
tmp_file.write(pdf_bytes)
|
| 165 |
+
filepath = tmp_file.name
|
| 166 |
+
return {'status': 'success', 'file': filepath, 'url': url}
|
| 167 |
+
|
| 168 |
+
except Exception as e_img:
|
| 169 |
+
import traceback
|
| 170 |
+
return {'status': 'error', 'message': f"Error procesando imagen para PDF: {str(e_img)}\n{traceback.format_exc()}", 'url': url}
|
| 171 |
+
|
| 172 |
+
# Procesamiento de texto para PDF
|
| 173 |
+
extracted_text_for_pdf = ""
|
| 174 |
+
if 'text/html' in content_type_or_error_msg and text_content:
|
| 175 |
+
soup = BeautifulSoup(text_content, 'html.parser')
|
| 176 |
+
for element in soup(["script", "style", "nav", "footer", "aside", "form", "button", "input", "header"]):
|
| 177 |
+
element.decompose()
|
| 178 |
+
main_content = soup.find('main') or soup.find('article') or soup.find('div', role='main') or soup.find('body')
|
| 179 |
+
if main_content:
|
| 180 |
+
text_items = [s.strip() for s in main_content.stripped_strings if s.strip()]
|
| 181 |
+
extracted_text_for_pdf = "\n".join(text_items)
|
| 182 |
+
else:
|
| 183 |
+
extracted_text_for_pdf = "\n".join([s.strip() for s in soup.stripped_strings if s.strip()])
|
| 184 |
+
|
| 185 |
+
elif 'text/plain' in content_type_or_error_msg and text_content:
|
| 186 |
+
extracted_text_for_pdf = text_content
|
| 187 |
+
elif text_content: # Otro tipo de contenido textual
|
| 188 |
+
extracted_text_for_pdf = text_content
|
| 189 |
+
else: # Error o tipo no textual no manejado como imagen
|
| 190 |
+
error_message = content_type_or_error_msg if isinstance(content_type_or_error_msg, str) else f"Tipo de contenido no soportado para PDF: {content_type_or_error_msg}"
|
| 191 |
+
return {'status': 'error', 'message': error_message, 'url': url}
|
| 192 |
|
| 193 |
+
if not extracted_text_for_pdf.strip():
|
| 194 |
+
return {'status': 'error', 'message': "No se encontró contenido textual para generar PDF.", 'url': url}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 195 |
|
| 196 |
+
try:
|
| 197 |
+
pdf = FPDF()
|
| 198 |
+
pdf.add_page()
|
| 199 |
+
pdf.set_auto_page_break(auto=True, margin=15)
|
| 200 |
+
|
| 201 |
+
if self.font_path:
|
| 202 |
+
pdf.add_font('DejaVu', '', self.font_path, uni=True)
|
| 203 |
+
current_font = 'DejaVu'
|
| 204 |
+
else:
|
| 205 |
+
current_font = 'Arial'
|
| 206 |
|
| 207 |
+
pdf.set_font(current_font, 'B', 12)
|
| 208 |
+
# FPDF no maneja bien URLs muy largas en write() directamente si contienen caracteres especiales.
|
| 209 |
+
# Mejor limpiar y escribir la URL.
|
| 210 |
+
# Usar multi_cell para la URL para permitir word wrapping si es muy larga.
|
| 211 |
+
pdf.multi_cell(0, 8, f"Contenido de: {url}")
|
| 212 |
+
pdf.ln(6) # Más pequeño que 10
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 213 |
|
| 214 |
+
pdf.set_font(current_font, '', 11)
|
| 215 |
+
|
| 216 |
+
clean_text = extracted_text_for_pdf.replace('\u2013', '-').replace('\u2014', '--')
|
| 217 |
+
clean_text = clean_text.replace('\u2018', "'").replace('\u2019', "'")
|
| 218 |
+
clean_text = clean_text.replace('\u201c', '"').replace('\u201d', '"')
|
| 219 |
+
clean_text = clean_text.replace('\u2026', '...')
|
| 220 |
+
clean_text = clean_text.replace('\u00A0', ' ') # Non-breaking space
|
| 221 |
+
|
| 222 |
+
printable_text = "".join(c for c in clean_text if c.isprintable() or c in ('\n', '\r', '\t'))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
|
| 224 |
+
# Dividir el texto en párrafos para evitar problemas con multi_cell y caracteres extraños.
|
| 225 |
+
paragraphs = printable_text.split('\n')
|
| 226 |
+
for para in paragraphs:
|
| 227 |
+
if para.strip(): # Solo procesar párrafos no vacíos
|
| 228 |
+
pdf.multi_cell(0, 7, para)
|
| 229 |
+
pdf.ln(2) # Pequeño espacio entre párrafos de multi_cell
|
| 230 |
+
else: # Si es un salto de línea intencional (párrafo vacío), añadir un pequeño ln
|
| 231 |
+
pdf.ln(5)
|
| 232 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 233 |
|
| 234 |
with tempfile.NamedTemporaryFile(delete=False, mode='wb', suffix='.pdf') as tmp_file:
|
| 235 |
+
pdf_output_bytes = pdf.output(dest='S').encode('latin-1') # FPDF output
|
| 236 |
tmp_file.write(pdf_output_bytes)
|
| 237 |
filepath = tmp_file.name
|
| 238 |
return {'status': 'success', 'file': filepath, 'url': url}
|
| 239 |
+
except Exception as e:
|
| 240 |
+
import traceback
|
| 241 |
tb_str = traceback.format_exc()
|
| 242 |
+
error_message = f"Error al generar PDF: {str(e)}\nDetalles: {tb_str}"
|
| 243 |
+
if len(error_message) > 500: error_message = error_message[:497] + "..."
|
|
|
|
| 244 |
return {'status': 'error', 'message': error_message, 'url': url}
|