Prakyath01 commited on
Commit
f91e586
·
verified ·
1 Parent(s): 16f9571

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +10 -0
app.py CHANGED
@@ -50,16 +50,26 @@ def scrape_page(name, url):
50
  r = requests.get(url, timeout=20)
51
  r.raise_for_status()
52
  soup = BeautifulSoup(r.text, "html.parser")
 
 
53
  content = soup.find("div", class_="td-content")
 
 
 
 
 
54
  if not content:
55
  return None
 
56
  text = content.get_text(separator="\n").strip()
57
  return Document(page_content=text, metadata={"doc_id": name, "url": url})
 
58
  except Exception as e:
59
  print(f"[ERROR] scraping {url}: {e}")
60
  return None
61
 
62
 
 
63
  def build_or_load_kb():
64
  embedding_model = HuggingFaceEmbeddings(
65
  model_name="sentence-transformers/all-MiniLM-L6-v2"
 
50
  r = requests.get(url, timeout=20)
51
  r.raise_for_status()
52
  soup = BeautifulSoup(r.text, "html.parser")
53
+
54
+ # Try Kubernetes docs structure
55
  content = soup.find("div", class_="td-content")
56
+
57
+ # Try Docker docs structure
58
+ if not content:
59
+ content = soup.find("div", class_="docs-content")
60
+
61
  if not content:
62
  return None
63
+
64
  text = content.get_text(separator="\n").strip()
65
  return Document(page_content=text, metadata={"doc_id": name, "url": url})
66
+
67
  except Exception as e:
68
  print(f"[ERROR] scraping {url}: {e}")
69
  return None
70
 
71
 
72
+
73
  def build_or_load_kb():
74
  embedding_model = HuggingFaceEmbeddings(
75
  model_name="sentence-transformers/all-MiniLM-L6-v2"