| from pathlib import Path |
| import json |
|
|
| import requests |
| from bs4 import BeautifulSoup |
|
|
| URL = "https://samagama.in/internship/faq" |
| OUT_FILE = Path("faq_kb.json") |
|
|
| def clean(text: str) -> str: |
| text = (text or "").replace("§", "") |
| text = " ".join(text.split()) |
| return text.strip() |
|
|
| def get_answer(detail) -> str: |
| data_text = detail.get("data-text") |
| if data_text: |
| return clean(data_text) |
|
|
| clone = BeautifulSoup(str(detail), "lxml").find("details") |
| summary = clone.find("summary") |
| if summary: |
| summary.decompose() |
| return clean(clone.get_text(" ", strip=True)) |
|
|
| def main(): |
| html = requests.get(URL, timeout=30).text |
| soup = BeautifulSoup(html, "lxml") |
|
|
| items = [] |
|
|
| for detail in soup.select("details.faq-q"): |
| qid = detail.get("id", "").strip() |
| summary = detail.find("summary") |
| if not qid or not summary: |
| continue |
|
|
| items.append({ |
| "id": qid, |
| "question": clean(summary.get_text(" ", strip=True)), |
| "answer": get_answer(detail), |
| "url": f"{URL}#{qid}", |
| }) |
|
|
| OUT_FILE.write_text(json.dumps(items, indent=2, ensure_ascii=False), encoding="utf-8") |
| print(f"Saved {len(items)} Q&A pairs to {OUT_FILE}") |
|
|
| if __name__ == "__main__": |
| main() |
