Subhajit Chakraborty commited on
Commit
91df0bf
·
1 Parent(s): 68fa393

update files

Browse files
Files changed (2) hide show
  1. src/app.py +1 -0
  2. src/services/scrape_worker.py +2 -2
src/app.py CHANGED
@@ -415,6 +415,7 @@ if st.session_state.page == "IntelliSCORE":
415
  res = lead_scorer.scrape_and_augment(
416
  additional_info, comp_url
417
  )
 
418
  with open(os.path.join(DATA_DIR, "lead_conditions.json"), "w") as f:
419
  json.dump(res, f, indent=2)
420
  st.success("Scrapping Completed!")
 
415
  res = lead_scorer.scrape_and_augment(
416
  additional_info, comp_url
417
  )
418
+ print(res)
419
  with open(os.path.join(DATA_DIR, "lead_conditions.json"), "w") as f:
420
  json.dump(res, f, indent=2)
421
  st.success("Scrapping Completed!")
src/services/scrape_worker.py CHANGED
@@ -8,9 +8,9 @@ def scrape_website(url: str) -> str:
8
  res = session.get(url, timeout=15)
9
  res.html.render(timeout=20)
10
  text = " ".join(res.html.text.split())
11
- return text[:8000]
12
  except Exception as e:
13
- return f"Error scraping the URL {url}: {e}"
14
  finally:
15
  session.close()
16
 
 
8
  res = session.get(url, timeout=15)
9
  res.html.render(timeout=20)
10
  text = " ".join(res.html.text.split())
11
+ return {"text": text[:8000]}
12
  except Exception as e:
13
+ return {"error": f"Scraping failed for {url}: {e}"}
14
  finally:
15
  session.close()
16