Spaces:
Sleeping
Sleeping
Removed string input from console
Browse files- code/web_scraper_allegro.py +1 -3
- code/web_scraper_ebay.py +2 -2
- code/web_scraper_olx.py +72 -31
code/web_scraper_allegro.py
CHANGED
|
@@ -63,9 +63,7 @@ def get_api_token():
|
|
| 63 |
print("Info: API Token loaded from .env file.")
|
| 64 |
return token
|
| 65 |
|
| 66 |
-
|
| 67 |
-
token = input("Please enter your Apify API Token: ").strip()
|
| 68 |
-
return token
|
| 69 |
|
| 70 |
def get_allegro_data(url):
|
| 71 |
apify_token = get_api_token()
|
|
|
|
| 63 |
print("Info: API Token loaded from .env file.")
|
| 64 |
return token
|
| 65 |
|
| 66 |
+
return AttributeError("API Token is required but not provided.")
|
|
|
|
|
|
|
| 67 |
|
| 68 |
def get_allegro_data(url):
|
| 69 |
apify_token = get_api_token()
|
code/web_scraper_ebay.py
CHANGED
|
@@ -66,8 +66,8 @@ def get_api_token():
|
|
| 66 |
print("Info: API Token loaded from .env file.")
|
| 67 |
return token
|
| 68 |
|
| 69 |
-
|
| 70 |
-
|
| 71 |
|
| 72 |
def get_ebay_data(url):
|
| 73 |
apify_token = get_api_token()
|
|
|
|
| 66 |
print("Info: API Token loaded from .env file.")
|
| 67 |
return token
|
| 68 |
|
| 69 |
+
return AttributeError("API Token is required but not provided.")
|
| 70 |
+
|
| 71 |
|
| 72 |
def get_ebay_data(url):
|
| 73 |
apify_token = get_api_token()
|
code/web_scraper_olx.py
CHANGED
|
@@ -1,55 +1,96 @@
|
|
| 1 |
-
# scrape_olx_offer.py
|
| 2 |
import requests
|
| 3 |
from bs4 import BeautifulSoup
|
|
|
|
| 4 |
|
| 5 |
-
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
headers = {
|
| 8 |
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
|
| 9 |
}
|
| 10 |
-
|
| 11 |
-
print(f"🔍 OLX: {url}")
|
| 12 |
response = requests.get(url, headers=headers)
|
| 13 |
-
|
| 14 |
if response.status_code != 200:
|
| 15 |
-
|
| 16 |
-
|
| 17 |
soup = BeautifulSoup(response.content, "html.parser")
|
| 18 |
-
|
| 19 |
-
#
|
|
|
|
|
|
|
| 20 |
title_element = soup.find("h4", class_="css-1au435n")
|
| 21 |
title = title_element.get_text().strip() if title_element else "untitled"
|
| 22 |
-
|
| 23 |
-
#
|
| 24 |
description_element = soup.find("div", class_="css-19duwlz")
|
| 25 |
-
description =
|
| 26 |
-
|
| 27 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
parameter_list = []
|
| 29 |
-
parameters_container = soup.find(
|
|
|
|
|
|
|
| 30 |
if parameters_container:
|
| 31 |
params = parameters_container.find_all("p", class_="css-13x8d99")
|
| 32 |
for p in params:
|
| 33 |
parameter_list.append(p.get_text().strip())
|
| 34 |
-
|
| 35 |
-
#
|
| 36 |
images = soup.select('img[data-testid^="swiper-image"]')
|
| 37 |
-
unique_links = set()
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
if link:
|
| 41 |
-
unique_links.add(link)
|
| 42 |
-
|
| 43 |
return {
|
| 44 |
-
"platform": "olx",
|
| 45 |
-
"url": url,
|
| 46 |
"title": title,
|
|
|
|
|
|
|
| 47 |
"description": description,
|
| 48 |
"parameters": parameter_list,
|
| 49 |
-
"image_urls":
|
|
|
|
| 50 |
}
|
| 51 |
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
|
|
|
|
|
|
|
|
| 1 |
import requests
|
| 2 |
from bs4 import BeautifulSoup
|
| 3 |
+
import json
|
| 4 |
|
| 5 |
+
|
| 6 |
+
# --- FUNCTIONS ---
|
| 7 |
+
def sanitize_name(text):
|
| 8 |
+
"""Sanitizes text by removing Polish characters and special symbols for a folder name."""
|
| 9 |
+
polish_chars = {
|
| 10 |
+
"ą": "a",
|
| 11 |
+
"ć": "c",
|
| 12 |
+
"ę": "e",
|
| 13 |
+
"ł": "l",
|
| 14 |
+
"ń": "n",
|
| 15 |
+
"ó": "o",
|
| 16 |
+
"ś": "s",
|
| 17 |
+
"ź": "z",
|
| 18 |
+
"ż": "z",
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
text = text.lower()
|
| 22 |
+
result = ""
|
| 23 |
+
|
| 24 |
+
for char in text:
|
| 25 |
+
if char in polish_chars:
|
| 26 |
+
result += polish_chars[char]
|
| 27 |
+
elif char.isalnum():
|
| 28 |
+
result += char
|
| 29 |
+
else:
|
| 30 |
+
result += "_"
|
| 31 |
+
|
| 32 |
+
# Remove double underscores
|
| 33 |
+
while "__" in result:
|
| 34 |
+
result = result.replace("__", "_")
|
| 35 |
+
|
| 36 |
+
return result.strip("_")
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def get_olx_data(url):
|
| 40 |
+
"""Fetches OLX offer data and returns it as a dictionary with title, description, parameters, and image URLs."""
|
| 41 |
+
# --- CONFIGURATION ---
|
| 42 |
headers = {
|
| 43 |
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
|
| 44 |
}
|
| 45 |
+
|
|
|
|
| 46 |
response = requests.get(url, headers=headers)
|
| 47 |
+
|
| 48 |
if response.status_code != 200:
|
| 49 |
+
return {"error": f"Connection error. Status code: {response.status_code}"}
|
| 50 |
+
|
| 51 |
soup = BeautifulSoup(response.content, "html.parser")
|
| 52 |
+
|
| 53 |
+
# --- GATHERING DATA ---
|
| 54 |
+
|
| 55 |
+
# Title
|
| 56 |
title_element = soup.find("h4", class_="css-1au435n")
|
| 57 |
title = title_element.get_text().strip() if title_element else "untitled"
|
| 58 |
+
|
| 59 |
+
# Description
|
| 60 |
description_element = soup.find("div", class_="css-19duwlz")
|
| 61 |
+
description = (
|
| 62 |
+
description_element.get_text(separator="\n").strip()
|
| 63 |
+
if description_element
|
| 64 |
+
else "No description"
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
# Parameters
|
| 68 |
parameter_list = []
|
| 69 |
+
parameters_container = soup.find(
|
| 70 |
+
"div", attrs={"data-testid": "ad-parameters-container"}
|
| 71 |
+
)
|
| 72 |
if parameters_container:
|
| 73 |
params = parameters_container.find_all("p", class_="css-13x8d99")
|
| 74 |
for p in params:
|
| 75 |
parameter_list.append(p.get_text().strip())
|
| 76 |
+
|
| 77 |
+
# Image Links
|
| 78 |
images = soup.select('img[data-testid^="swiper-image"]')
|
| 79 |
+
unique_links = list(set(img.get("src") for img in images if img.get("src")))
|
| 80 |
+
|
| 81 |
+
# --- RETURNING DICTIONARY ---
|
|
|
|
|
|
|
|
|
|
| 82 |
return {
|
|
|
|
|
|
|
| 83 |
"title": title,
|
| 84 |
+
"sanitized_title": sanitize_name(title),
|
| 85 |
+
"url": url,
|
| 86 |
"description": description,
|
| 87 |
"parameters": parameter_list,
|
| 88 |
+
"image_urls": unique_links,
|
| 89 |
+
"image_count": len(unique_links),
|
| 90 |
}
|
| 91 |
|
| 92 |
+
|
| 93 |
+
# --- USAGE ---
|
| 94 |
+
link = input("Enter the OLX offer link: ")
|
| 95 |
+
data = get_olx_data(link)
|
| 96 |
+
print(json.dumps(data, indent=4, ensure_ascii=False))
|