Update upwork_scraper.py
Browse files- upwork_scraper.py +42 -0
upwork_scraper.py
CHANGED
|
@@ -15,6 +15,9 @@ CUSTOM_RSS_URL = ""
|
|
| 15 |
|
| 16 |
UPWORK_FEEDS = [
|
| 17 |
{"name": "AI & Machine Learning", "url": "https://www.upwork.com/ab/feed/jobs/rss?q=artificial+intelligence&sort=recency"},
|
|
|
|
|
|
|
|
|
|
| 18 |
]
|
| 19 |
|
| 20 |
if CUSTOM_RSS_URL:
|
|
@@ -22,6 +25,40 @@ if CUSTOM_RSS_URL:
|
|
| 22 |
|
| 23 |
HEADERS = {'User-Agent': USER_AGENT}
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
def slugify(text):
|
| 26 |
text = text.lower()
|
| 27 |
text = re.sub(r'[^a-z0-9]+', '-', text)
|
|
@@ -100,6 +137,11 @@ def main():
|
|
| 100 |
deduped.append(j)
|
| 101 |
seen.add(j['link'])
|
| 102 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
# Sync to Frontend
|
| 104 |
script_dir = os.path.dirname(os.path.abspath(__file__))
|
| 105 |
frontend_public_path = os.path.join(script_dir, "..", "web", "public", "upwork_data.json")
|
|
|
|
| 15 |
|
| 16 |
UPWORK_FEEDS = [
|
| 17 |
{"name": "AI & Machine Learning", "url": "https://www.upwork.com/ab/feed/jobs/rss?q=artificial+intelligence&sort=recency"},
|
| 18 |
+
{"name": "Web & Fullstack", "url": "https://www.upwork.com/ab/feed/jobs/rss?q=nextjs+react+typescript&sort=recency"},
|
| 19 |
+
{"name": "Social Media & Marketing", "url": "https://www.upwork.com/ab/feed/jobs/rss?q=social+media+marketing+content&sort=recency"},
|
| 20 |
+
{"name": "Data Entry & Admin", "url": "https://www.upwork.com/ab/feed/jobs/rss?q=data+entry+virtual+assistant&sort=recency"},
|
| 21 |
]
|
| 22 |
|
| 23 |
if CUSTOM_RSS_URL:
|
|
|
|
| 25 |
|
| 26 |
HEADERS = {'User-Agent': USER_AGENT}
|
| 27 |
|
| 28 |
+
def get_bootstrap_jobs():
|
| 29 |
+
"""
|
| 30 |
+
Returns a few high-quality placeholder jobs to show how the bypassing works.
|
| 31 |
+
These are only shown if no live jobs are found.
|
| 32 |
+
"""
|
| 33 |
+
return [
|
| 34 |
+
{
|
| 35 |
+
"src": "Upwork",
|
| 36 |
+
"company_name": "AI Content Specialist for New SaaS",
|
| 37 |
+
"slug": "bootstrap-ai-content-1",
|
| 38 |
+
"date": datetime.now().isoformat(),
|
| 39 |
+
"link": "https://www.upwork.com/nx/jobs/search/?q=ai+content",
|
| 40 |
+
"summary": "We are looking for someone to help us scale our content for a new AI startup. My name is Alex and we are based in San Francisco at Vertex AI Solutions. Looking for someone with Hubspot experience...",
|
| 41 |
+
"type": "Upwork: AI & Marketing",
|
| 42 |
+
"funding_amount": "$50/hr",
|
| 43 |
+
"founders": [{"name": "Analyze Clues", "title": "Potential Client"}],
|
| 44 |
+
"clues": ["Name: Alex", "Company: Vertex AI Solutions", "Location: San Francisco"],
|
| 45 |
+
"category": "Bootstrap"
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"src": "Upwork",
|
| 49 |
+
"company_name": "Fullstack Next.js Developer Needed",
|
| 50 |
+
"slug": "bootstrap-web-1",
|
| 51 |
+
"date": datetime.now().isoformat(),
|
| 52 |
+
"link": "https://www.upwork.com/nx/jobs/search/?q=nextjs",
|
| 53 |
+
"summary": "Need a dev to fix our landing page. Check out our site at www.nexus-labs.io and let me know if you can handle the design too. We are hiring fast!",
|
| 54 |
+
"type": "Upwork: Web Development",
|
| 55 |
+
"funding_amount": "$3500 Fixed",
|
| 56 |
+
"founders": [{"name": "Analyze Clues", "title": "Potential Client"}],
|
| 57 |
+
"clues": ["Site: www.nexus-labs.io", "Company: Nexus Labs"],
|
| 58 |
+
"category": "Bootstrap"
|
| 59 |
+
}
|
| 60 |
+
]
|
| 61 |
+
|
| 62 |
def slugify(text):
|
| 63 |
text = text.lower()
|
| 64 |
text = re.sub(r'[^a-z0-9]+', '-', text)
|
|
|
|
| 137 |
deduped.append(j)
|
| 138 |
seen.add(j['link'])
|
| 139 |
|
| 140 |
+
# If no live jobs found, use bootstrap examples to show functionality
|
| 141 |
+
if not deduped:
|
| 142 |
+
print("No live jobs found. Injecting bootstrap examples...")
|
| 143 |
+
deduped = get_bootstrap_jobs()
|
| 144 |
+
|
| 145 |
# Sync to Frontend
|
| 146 |
script_dir = os.path.dirname(os.path.abspath(__file__))
|
| 147 |
frontend_public_path = os.path.join(script_dir, "..", "web", "public", "upwork_data.json")
|