Fix scraper CSS selectors and add error logging
Browse files- backend/cron.py +1 -1
- backend/main.py +1 -1
- backend/scrapers/nigeria.py +11 -2
backend/cron.py
CHANGED
|
@@ -39,7 +39,7 @@ def _scrape_board(board_name: str, cat: str, query: str) -> list[dict]:
|
|
| 39 |
j["category"] = cat
|
| 40 |
return result
|
| 41 |
except Exception as e:
|
| 42 |
-
|
| 43 |
return []
|
| 44 |
|
| 45 |
|
|
|
|
| 39 |
j["category"] = cat
|
| 40 |
return result
|
| 41 |
except Exception as e:
|
| 42 |
+
print(f"[cron] Error scraping {board_name} ({cat}/{query}): {e}")
|
| 43 |
return []
|
| 44 |
|
| 45 |
|
backend/main.py
CHANGED
|
@@ -272,7 +272,7 @@ def _scrape_board(board_name: str, cat: str, query: str) -> list[dict]:
|
|
| 272 |
j["source"] = board_name; j["category"] = cat
|
| 273 |
return result
|
| 274 |
except Exception as e:
|
| 275 |
-
|
| 276 |
return []
|
| 277 |
|
| 278 |
|
|
|
|
| 272 |
j["source"] = board_name; j["category"] = cat
|
| 273 |
return result
|
| 274 |
except Exception as e:
|
| 275 |
+
print(f"[scrape] Error scraping {board_name} ({cat}): {e}")
|
| 276 |
return []
|
| 277 |
|
| 278 |
|
backend/scrapers/nigeria.py
CHANGED
|
@@ -10,10 +10,11 @@ CAT = "nigeria"
|
|
| 10 |
_TITLE_SELS = ["h2 a","h3 a","h4 a","a.job-title","[class*='title'] a",
|
| 11 |
"a[class*='title']","a[class*='job']",".job-name a","a"]
|
| 12 |
_COMPANY_SELS = ["[class*='company'] a","[class*='company']","[class*='employer']",
|
| 13 |
-
"[class*='recruiter']","[class*='org']","span[class*='name']"
|
|
|
|
| 14 |
_LOCATION_SELS = ["[class*='location']","[class*='place']","[class*='city']",
|
| 15 |
"[class*='state']","span[class*='loc']","[class*='area']"]
|
| 16 |
-
_DATE_SELS = ["time","[class*='date']","[class*='posted']","[class*='time']","span.date"]
|
| 17 |
_DESC_SELS = ["[class*='desc']","[class*='excerpt']","[class*='summary']","p"]
|
| 18 |
|
| 19 |
|
|
@@ -62,7 +63,9 @@ class NigerianJobScraper(BaseScraper):
|
|
| 62 |
if ld:
|
| 63 |
for j in ld: j.update({"source": "myjobmag", "category": CAT})
|
| 64 |
return self.filter_fresh(ld[:20])
|
|
|
|
| 65 |
cards = self.multi_select(soup, [
|
|
|
|
| 66 |
"article.job-item","div.job-item","li.job-item",
|
| 67 |
"div[class*='job-list'] > div","ul.jobs-list > li",
|
| 68 |
"div[class*='joblist'] article","article",
|
|
@@ -103,6 +106,7 @@ class NigerianJobScraper(BaseScraper):
|
|
| 103 |
for j in ld: j.update({"source": "jobgurus", "category": CAT})
|
| 104 |
return self.filter_fresh(ld[:15])
|
| 105 |
cards = self.multi_select(soup, [
|
|
|
|
| 106 |
"article.job_listing","article[class*='job']",
|
| 107 |
"li.job_listing","div.job_listing","article","div[class*='listing']",
|
| 108 |
])
|
|
@@ -160,6 +164,7 @@ class NigerianJobScraper(BaseScraper):
|
|
| 160 |
print(f"[jobzilla] {len(ld)} jobs for '{query}'")
|
| 161 |
return self.filter_fresh(ld[:20])
|
| 162 |
cards = self.multi_select(soup, [
|
|
|
|
| 163 |
"div.job-item","div.job-listing","div.search-result","div.job-row",
|
| 164 |
"div.job-card","article.job","li.job-item","div[class*='job']","article",
|
| 165 |
])
|
|
@@ -178,6 +183,10 @@ class NigerianJobScraper(BaseScraper):
|
|
| 178 |
if not title or not href or href in seen: continue
|
| 179 |
seen.add(href)
|
| 180 |
company = self.first_text(card, _COMPANY_SELS)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
location = self.first_text(card, _LOCATION_SELS, "Nigeria")
|
| 182 |
posted = self.first_text(card, _DATE_SELS)
|
| 183 |
desc = self.first_text(card, _DESC_SELS)
|
|
|
|
| 10 |
_TITLE_SELS = ["h2 a","h3 a","h4 a","a.job-title","[class*='title'] a",
|
| 11 |
"a[class*='title']","a[class*='job']",".job-name a","a"]
|
| 12 |
_COMPANY_SELS = ["[class*='company'] a","[class*='company']","[class*='employer']",
|
| 13 |
+
"[class*='recruiter']","[class*='org']","span[class*='name']",
|
| 14 |
+
"[class*='logo'] a img","[class*='logo'] img"]
|
| 15 |
_LOCATION_SELS = ["[class*='location']","[class*='place']","[class*='city']",
|
| 16 |
"[class*='state']","span[class*='loc']","[class*='area']"]
|
| 17 |
+
_DATE_SELS = ["time","[class*='date']","[class*='posted']","[class*='time']","span.date","#job-date"]
|
| 18 |
_DESC_SELS = ["[class*='desc']","[class*='excerpt']","[class*='summary']","p"]
|
| 19 |
|
| 20 |
|
|
|
|
| 63 |
if ld:
|
| 64 |
for j in ld: j.update({"source": "myjobmag", "category": CAT})
|
| 65 |
return self.filter_fresh(ld[:20])
|
| 66 |
+
# Try common card selectors first, then site-specific
|
| 67 |
cards = self.multi_select(soup, [
|
| 68 |
+
"li.job-list-li",
|
| 69 |
"article.job-item","div.job-item","li.job-item",
|
| 70 |
"div[class*='job-list'] > div","ul.jobs-list > li",
|
| 71 |
"div[class*='joblist'] article","article",
|
|
|
|
| 106 |
for j in ld: j.update({"source": "jobgurus", "category": CAT})
|
| 107 |
return self.filter_fresh(ld[:15])
|
| 108 |
cards = self.multi_select(soup, [
|
| 109 |
+
"div.panel.panel-default.job-post-panel",
|
| 110 |
"article.job_listing","article[class*='job']",
|
| 111 |
"li.job_listing","div.job_listing","article","div[class*='listing']",
|
| 112 |
])
|
|
|
|
| 164 |
print(f"[jobzilla] {len(ld)} jobs for '{query}'")
|
| 165 |
return self.filter_fresh(ld[:20])
|
| 166 |
cards = self.multi_select(soup, [
|
| 167 |
+
"div.card.border-0.shadow",
|
| 168 |
"div.job-item","div.job-listing","div.search-result","div.job-row",
|
| 169 |
"div.job-card","article.job","li.job-item","div[class*='job']","article",
|
| 170 |
])
|
|
|
|
| 183 |
if not title or not href or href in seen: continue
|
| 184 |
seen.add(href)
|
| 185 |
company = self.first_text(card, _COMPANY_SELS)
|
| 186 |
+
if not company:
|
| 187 |
+
logo = card.select_one("[class*='logo'] img")
|
| 188 |
+
if logo and logo.get("alt"):
|
| 189 |
+
company = logo["alt"]
|
| 190 |
location = self.first_text(card, _LOCATION_SELS, "Nigeria")
|
| 191 |
posted = self.first_text(card, _DATE_SELS)
|
| 192 |
desc = self.first_text(card, _DESC_SELS)
|