File size: 1,296 Bytes
e92d49a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | from pathlib import Path
import json
import requests
from bs4 import BeautifulSoup
URL = "https://samagama.in/internship/faq"
OUT_FILE = Path("faq_kb.json")
def clean(text: str) -> str:
text = (text or "").replace("§", "")
text = " ".join(text.split())
return text.strip()
def get_answer(detail) -> str:
data_text = detail.get("data-text")
if data_text:
return clean(data_text)
clone = BeautifulSoup(str(detail), "lxml").find("details")
summary = clone.find("summary")
if summary:
summary.decompose()
return clean(clone.get_text(" ", strip=True))
def main():
html = requests.get(URL, timeout=30).text
soup = BeautifulSoup(html, "lxml")
items = []
for detail in soup.select("details.faq-q"):
qid = detail.get("id", "").strip()
summary = detail.find("summary")
if not qid or not summary:
continue
items.append({
"id": qid,
"question": clean(summary.get_text(" ", strip=True)),
"answer": get_answer(detail),
"url": f"{URL}#{qid}",
})
OUT_FILE.write_text(json.dumps(items, indent=2, ensure_ascii=False), encoding="utf-8")
print(f"Saved {len(items)} Q&A pairs to {OUT_FILE}")
if __name__ == "__main__":
main() |