File size: 2,175 Bytes
11ba2bd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | """Parse the FreeCAD-documentation repo into a list of page dicts."""
import os
import re
from pathlib import Path
from typing import Iterator
from src.config import WIKI_BASE_URL
_SKIP_PATTERNS = re.compile(
r"(Category:|File:|Template:|Special:|MediaWiki:|User:|Talk:|^index\.md$)",
re.IGNORECASE,
)
_PRIORITY_PAGES = {
"Python_scripting_tutorial", "FreeCAD_Scripting_Basics", "Scripting_and_macros",
"Part_scripting", "Sketcher_scripting", "PartDesign_scripting",
"Topological_naming_problem", "Scripted_objects", "Scripted_objects_migration",
"PartDesign_Pad", "PartDesign_Pocket", "PartDesign_Revolution", "PartDesign_Body",
"PartDesign_Fillet", "PartDesign_Chamfer", "PartDesign_Hole", "PartDesign_Boolean",
"PartDesign_AdditiveLoft", "PartDesign_AdditivePipe", "PartDesign_Workbench",
"Sketcher_Workbench", "Release_notes_1.0", "Release_notes_1.1",
"Basic_Part_Design_Tutorial_019", "Creating_a_simple_part_with_PartDesign",
"Spreadsheet_Workbench",
}
def _page_title(stem: str) -> str:
return stem.replace("_", " ")
def _source_url(stem: str) -> str:
return f"{WIKI_BASE_URL}/{stem}"
def iter_pages(wiki_dir: str | Path) -> Iterator[dict]:
wiki_path = Path(wiki_dir)
for md_file in sorted(wiki_path.glob("*.md")):
stem = md_file.stem
if _SKIP_PATTERNS.search(stem):
continue
raw = md_file.read_text(encoding="utf-8", errors="replace")
if len(raw.strip()) < 200:
continue
yield {
"source_file": str(md_file),
"page_title": _page_title(stem),
"source_url": _source_url(stem),
"raw_text": raw,
"priority": stem in _PRIORITY_PAGES,
}
def load_freecad_docs(repo_root: str) -> list[dict]:
wiki_dir = os.path.join(repo_root, "wiki")
if not os.path.isdir(wiki_dir):
raise FileNotFoundError(
f"Expected wiki/ directory at {wiki_dir}. "
"Clone https://github.com/FreeCAD/FreeCAD-documentation first."
)
pages = list(iter_pages(wiki_dir))
pages.sort(key=lambda p: (not p["priority"], p["page_title"]))
return pages
|