hatamo commited on
Commit
567001c
·
1 Parent(s): f171fee

Removed string input from console

Browse files
code/web_scraper_allegro.py CHANGED
@@ -63,9 +63,7 @@ def get_api_token():
63
  print("Info: API Token loaded from .env file.")
64
  return token
65
 
66
- print("Warning: APIFY_TOKEN not found in .env file.")
67
- token = input("Please enter your Apify API Token: ").strip()
68
- return token
69
 
70
  def get_allegro_data(url):
71
  apify_token = get_api_token()
 
63
  print("Info: API Token loaded from .env file.")
64
  return token
65
 
66
+ return AttributeError("API Token is required but not provided.")
 
 
67
 
68
  def get_allegro_data(url):
69
  apify_token = get_api_token()
code/web_scraper_ebay.py CHANGED
@@ -66,8 +66,8 @@ def get_api_token():
66
  print("Info: API Token loaded from .env file.")
67
  return token
68
 
69
- print("Warning: APIFY_TOKEN not found in .env file.")
70
- return input("Please enter your Apify API Token: ").strip()
71
 
72
  def get_ebay_data(url):
73
  apify_token = get_api_token()
 
66
  print("Info: API Token loaded from .env file.")
67
  return token
68
 
69
+ return AttributeError("API Token is required but not provided.")
70
+
71
 
72
  def get_ebay_data(url):
73
  apify_token = get_api_token()
code/web_scraper_olx.py CHANGED
@@ -1,55 +1,96 @@
1
- # scrape_olx_offer.py
2
  import requests
3
  from bs4 import BeautifulSoup
 
4
 
5
- def scrape_olx_offer(url: str):
6
- """Zwraca dane aukcji bez zapisywania na dysk"""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  headers = {
8
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
9
  }
10
-
11
- print(f"🔍 OLX: {url}")
12
  response = requests.get(url, headers=headers)
13
-
14
  if response.status_code != 200:
15
- raise ValueError(f"OLX error: {response.status_code}")
16
-
17
  soup = BeautifulSoup(response.content, "html.parser")
18
-
19
- # TITLE
 
 
20
  title_element = soup.find("h4", class_="css-1au435n")
21
  title = title_element.get_text().strip() if title_element else "untitled"
22
-
23
- # DESCRIPTION
24
  description_element = soup.find("div", class_="css-19duwlz")
25
- description = description_element.get_text(separator="\n").strip() if description_element else "No description"
26
-
27
- # PARAMETERS
 
 
 
 
28
  parameter_list = []
29
- parameters_container = soup.find("div", attrs={"data-testid": "ad-parameters-container"})
 
 
30
  if parameters_container:
31
  params = parameters_container.find_all("p", class_="css-13x8d99")
32
  for p in params:
33
  parameter_list.append(p.get_text().strip())
34
-
35
- # IMAGES
36
  images = soup.select('img[data-testid^="swiper-image"]')
37
- unique_links = set()
38
- for img in images:
39
- link = img.get("src")
40
- if link:
41
- unique_links.add(link)
42
-
43
  return {
44
- "platform": "olx",
45
- "url": url,
46
  "title": title,
 
 
47
  "description": description,
48
  "parameters": parameter_list,
49
- "image_urls": list(unique_links)
 
50
  }
51
 
52
- if __name__ == "__main__":
53
- url = input("OLX URL: ")
54
- result = scrape_olx_offer(url)
55
- print(result)
 
 
 
1
  import requests
2
  from bs4 import BeautifulSoup
3
+ import json
4
 
5
+
6
+ # --- FUNCTIONS ---
7
+ def sanitize_name(text):
8
+ """Sanitizes text by removing Polish characters and special symbols for a folder name."""
9
+ polish_chars = {
10
+ "ą": "a",
11
+ "ć": "c",
12
+ "ę": "e",
13
+ "ł": "l",
14
+ "ń": "n",
15
+ "ó": "o",
16
+ "ś": "s",
17
+ "ź": "z",
18
+ "ż": "z",
19
+ }
20
+
21
+ text = text.lower()
22
+ result = ""
23
+
24
+ for char in text:
25
+ if char in polish_chars:
26
+ result += polish_chars[char]
27
+ elif char.isalnum():
28
+ result += char
29
+ else:
30
+ result += "_"
31
+
32
+ # Remove double underscores
33
+ while "__" in result:
34
+ result = result.replace("__", "_")
35
+
36
+ return result.strip("_")
37
+
38
+
39
+ def get_olx_data(url):
40
+ """Fetches OLX offer data and returns it as a dictionary with title, description, parameters, and image URLs."""
41
+ # --- CONFIGURATION ---
42
  headers = {
43
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
44
  }
45
+
 
46
  response = requests.get(url, headers=headers)
47
+
48
  if response.status_code != 200:
49
+ return {"error": f"Connection error. Status code: {response.status_code}"}
50
+
51
  soup = BeautifulSoup(response.content, "html.parser")
52
+
53
+ # --- GATHERING DATA ---
54
+
55
+ # Title
56
  title_element = soup.find("h4", class_="css-1au435n")
57
  title = title_element.get_text().strip() if title_element else "untitled"
58
+
59
+ # Description
60
  description_element = soup.find("div", class_="css-19duwlz")
61
+ description = (
62
+ description_element.get_text(separator="\n").strip()
63
+ if description_element
64
+ else "No description"
65
+ )
66
+
67
+ # Parameters
68
  parameter_list = []
69
+ parameters_container = soup.find(
70
+ "div", attrs={"data-testid": "ad-parameters-container"}
71
+ )
72
  if parameters_container:
73
  params = parameters_container.find_all("p", class_="css-13x8d99")
74
  for p in params:
75
  parameter_list.append(p.get_text().strip())
76
+
77
+ # Image Links
78
  images = soup.select('img[data-testid^="swiper-image"]')
79
+ unique_links = list(set(img.get("src") for img in images if img.get("src")))
80
+
81
+ # --- RETURNING DICTIONARY ---
 
 
 
82
  return {
 
 
83
  "title": title,
84
+ "sanitized_title": sanitize_name(title),
85
+ "url": url,
86
  "description": description,
87
  "parameters": parameter_list,
88
+ "image_urls": unique_links,
89
+ "image_count": len(unique_links),
90
  }
91
 
92
+
93
+ # --- USAGE ---
94
+ link = input("Enter the OLX offer link: ")
95
+ data = get_olx_data(link)
96
+ print(json.dumps(data, indent=4, ensure_ascii=False))