Britzzy commited on
Commit
2cc4426
·
1 Parent(s): ca76373

Fix scraper CSS selectors and add error logging

Browse files
backend/cron.py CHANGED
@@ -39,7 +39,7 @@ def _scrape_board(board_name: str, cat: str, query: str) -> list[dict]:
39
  j["category"] = cat
40
  return result
41
  except Exception as e:
42
- pass
43
  return []
44
 
45
 
 
39
  j["category"] = cat
40
  return result
41
  except Exception as e:
42
+ print(f"[cron] Error scraping {board_name} ({cat}/{query}): {e}")
43
  return []
44
 
45
 
backend/main.py CHANGED
@@ -272,7 +272,7 @@ def _scrape_board(board_name: str, cat: str, query: str) -> list[dict]:
272
  j["source"] = board_name; j["category"] = cat
273
  return result
274
  except Exception as e:
275
- pass
276
  return []
277
 
278
 
 
272
  j["source"] = board_name; j["category"] = cat
273
  return result
274
  except Exception as e:
275
+ print(f"[scrape] Error scraping {board_name} ({cat}): {e}")
276
  return []
277
 
278
 
backend/scrapers/nigeria.py CHANGED
@@ -10,10 +10,11 @@ CAT = "nigeria"
10
  _TITLE_SELS = ["h2 a","h3 a","h4 a","a.job-title","[class*='title'] a",
11
  "a[class*='title']","a[class*='job']",".job-name a","a"]
12
  _COMPANY_SELS = ["[class*='company'] a","[class*='company']","[class*='employer']",
13
- "[class*='recruiter']","[class*='org']","span[class*='name']"]
 
14
  _LOCATION_SELS = ["[class*='location']","[class*='place']","[class*='city']",
15
  "[class*='state']","span[class*='loc']","[class*='area']"]
16
- _DATE_SELS = ["time","[class*='date']","[class*='posted']","[class*='time']","span.date"]
17
  _DESC_SELS = ["[class*='desc']","[class*='excerpt']","[class*='summary']","p"]
18
 
19
 
@@ -62,7 +63,9 @@ class NigerianJobScraper(BaseScraper):
62
  if ld:
63
  for j in ld: j.update({"source": "myjobmag", "category": CAT})
64
  return self.filter_fresh(ld[:20])
 
65
  cards = self.multi_select(soup, [
 
66
  "article.job-item","div.job-item","li.job-item",
67
  "div[class*='job-list'] > div","ul.jobs-list > li",
68
  "div[class*='joblist'] article","article",
@@ -103,6 +106,7 @@ class NigerianJobScraper(BaseScraper):
103
  for j in ld: j.update({"source": "jobgurus", "category": CAT})
104
  return self.filter_fresh(ld[:15])
105
  cards = self.multi_select(soup, [
 
106
  "article.job_listing","article[class*='job']",
107
  "li.job_listing","div.job_listing","article","div[class*='listing']",
108
  ])
@@ -160,6 +164,7 @@ class NigerianJobScraper(BaseScraper):
160
  print(f"[jobzilla] {len(ld)} jobs for '{query}'")
161
  return self.filter_fresh(ld[:20])
162
  cards = self.multi_select(soup, [
 
163
  "div.job-item","div.job-listing","div.search-result","div.job-row",
164
  "div.job-card","article.job","li.job-item","div[class*='job']","article",
165
  ])
@@ -178,6 +183,10 @@ class NigerianJobScraper(BaseScraper):
178
  if not title or not href or href in seen: continue
179
  seen.add(href)
180
  company = self.first_text(card, _COMPANY_SELS)
 
 
 
 
181
  location = self.first_text(card, _LOCATION_SELS, "Nigeria")
182
  posted = self.first_text(card, _DATE_SELS)
183
  desc = self.first_text(card, _DESC_SELS)
 
10
  _TITLE_SELS = ["h2 a","h3 a","h4 a","a.job-title","[class*='title'] a",
11
  "a[class*='title']","a[class*='job']",".job-name a","a"]
12
  _COMPANY_SELS = ["[class*='company'] a","[class*='company']","[class*='employer']",
13
+ "[class*='recruiter']","[class*='org']","span[class*='name']",
14
+ "[class*='logo'] a img","[class*='logo'] img"]
15
  _LOCATION_SELS = ["[class*='location']","[class*='place']","[class*='city']",
16
  "[class*='state']","span[class*='loc']","[class*='area']"]
17
+ _DATE_SELS = ["time","[class*='date']","[class*='posted']","[class*='time']","span.date","#job-date"]
18
  _DESC_SELS = ["[class*='desc']","[class*='excerpt']","[class*='summary']","p"]
19
 
20
 
 
63
  if ld:
64
  for j in ld: j.update({"source": "myjobmag", "category": CAT})
65
  return self.filter_fresh(ld[:20])
66
+ # Try common card selectors first, then site-specific
67
  cards = self.multi_select(soup, [
68
+ "li.job-list-li",
69
  "article.job-item","div.job-item","li.job-item",
70
  "div[class*='job-list'] > div","ul.jobs-list > li",
71
  "div[class*='joblist'] article","article",
 
106
  for j in ld: j.update({"source": "jobgurus", "category": CAT})
107
  return self.filter_fresh(ld[:15])
108
  cards = self.multi_select(soup, [
109
+ "div.panel.panel-default.job-post-panel",
110
  "article.job_listing","article[class*='job']",
111
  "li.job_listing","div.job_listing","article","div[class*='listing']",
112
  ])
 
164
  print(f"[jobzilla] {len(ld)} jobs for '{query}'")
165
  return self.filter_fresh(ld[:20])
166
  cards = self.multi_select(soup, [
167
+ "div.card.border-0.shadow",
168
  "div.job-item","div.job-listing","div.search-result","div.job-row",
169
  "div.job-card","article.job","li.job-item","div[class*='job']","article",
170
  ])
 
183
  if not title or not href or href in seen: continue
184
  seen.add(href)
185
  company = self.first_text(card, _COMPANY_SELS)
186
+ if not company:
187
+ logo = card.select_one("[class*='logo'] img")
188
+ if logo and logo.get("alt"):
189
+ company = logo["alt"]
190
  location = self.first_text(card, _LOCATION_SELS, "Nigeria")
191
  posted = self.first_text(card, _DATE_SELS)
192
  desc = self.first_text(card, _DESC_SELS)