metadata_extractor.py

from typing import List, Dict, Any


def _walk(
    section: Dict[str, Any],
    doc_name: str,
    doc_description: str,
    level: int,
    path: List[str],
    entries: List[Dict[str, Any]],
):
    children = section.get("nodes", [])
    has_children = bool(children)
    title = section.get("title", "")
    hierarchy_path = path + [title] if title else path

    if has_children:
        summary_text = section.get("summary", "").strip()
        summary_entry = {
            "doc_name": doc_name,
            "doc_description": doc_description,
            "title": title,
            "node_id": section.get("node_id", ""),
            "text": summary_text,
            "summary": summary_text,
            "level": level,
            "hierarchy_path": hierarchy_path,
            "is_container": True,
            "is_leaf": False,
        }
        entries.append(summary_entry)

        for child in children:
            _walk(child, doc_name, doc_description, level + 1, hierarchy_path, entries)
    else:
        full_text = section.get("text", "")
        if not full_text.strip():
            return
        entries.append(
            {
                "doc_name": doc_name,
                "doc_description": doc_description,
                "title": title,
                "node_id": section.get("node_id", ""),
                "text": full_text,
                "summary": section.get("summary", ""),
                "level": level,
                "hierarchy_path": hierarchy_path,
                "is_container": False,
                "is_leaf": True,
            }
        )


def extract_metadata(doc: Dict[str, Any]) -> List[Dict[str, Any]]:
    entries: List[Dict[str, Any]] = []
    doc_name = doc.get("doc_name", "unknown")
    doc_description = doc.get("doc_description", "")

    for section in doc.get("structure", []):
        _walk(section, doc_name, doc_description, 0, [], entries)

    return entries