""" Build the `professional_codes` corpus domain. Council/board advertising standards for the audience's regulated professions: - Chiropractic Board — Advertising Policy 2025 - Osteopathic Council — Code of Conduct (Jan 2023, contains advertising provisions) - Physiotherapy Board — Code of Ethics and Professional Conduct (Code, Standards, Thresholds) - Chinese Medicine Council — Advertising Standard Guidance - Medical Council — Statement on Advertising (BENCHMARK ONLY — does NOT bind non-MD practitioners) The critical v2 design feature in this domain is `binds:` scope metadata. Each council document declares which practitioner classes it binds. The Medical Council statement is included as a benchmark but tagged `benchmark-only: true` so the retriever does not cite it as authoritative for chiropractors, naturopaths, etc. Becki's correction in v1 → v2 review: "the Medical Council statement is the strictest in the set, particularly on testimonials. For a chiropractor or naturopath user, citing it could give answers more conservative than their own regulator requires." Pattern adapted from `build_advertising_standards_compilation.py` (no LEGISLATION_SOURCES — all PDFs). """ from __future__ import annotations import os import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from clean_artifacts import clean_corpus_artifacts, format_stats # noqa: E402 from extract_pdf import extract_to_markdown, demote_headings # noqa: E402 PROJECT_ROOT = Path(__file__).resolve().parents[1] CORPUS_DIR = PROJECT_ROOT / "corpus" SOURCES_RAW = PROJECT_ROOT / "sources" / "raw" CORPUS_DIR.mkdir(exist_ok=True) SOURCES_RAW.mkdir(parents=True, exist_ok=True) DOMAIN_FILE = CORPUS_DIR / "professional-codes.md" GUIDANCE_SOURCES = [ { "name": "Chiropractic Board — Advertising Policy 2025", "url": "https://chiropracticboard.org.nz/assets/ElementalFiles/Policies/Advertising-Policy-2025-FINAL.pdf", "filename": "chiropractic-board-advertising-policy-2025.pdf", "format": "pdf", "section_title": "Chiropractic Board of NZ — Advertising Policy 2025", "metadata": { "binds": "registered chiropractors (NZ)", "issued_by": "Chiropractic Board of New Zealand (Te Poari Kaikorohiti o Aotearoa)", "applies_to": "advertising of chiropractic services and chiropractic care products", }, }, { "name": "Osteopathic Council — Code of Conduct for Osteopaths (Jan 2023)", "url": "https://www.osteopathiccouncil.org.nz/common/Uploaded%20files/Publications/OCNZ%20Code%20of%20Conduct%20Jan23.pdf", "filename": "osteopathic-council-code-of-conduct-jan-2023.pdf", "format": "pdf", "section_title": "Osteopathic Council of NZ — Code of Conduct (Jan 2023)", "metadata": { "binds": "registered osteopaths (NZ)", "issued_by": "Osteopathic Council of New Zealand", "applies_to": "professional conduct including advertising of osteopathic services", "notes": "the council's Code of Conduct includes advertising-specific provisions; for narrower advertising-only guidance see also OCNZ Medical Advertisement Policy 2016 (not currently included in v1)", }, }, { # The PDF is a 361-KB kitchen-sink compendium ("Standards framework") covering # clinical, cultural, and ethical standards — only ~30 KB is actually about # advertising. We slice out just the dedicated "Advertising standard" # subdocument (between "Advertising standard" and the next "Telehealth standard" # heading) to keep the corpus focused. "name": "Physiotherapy Board — Advertising standard (extracted from Standards framework)", "url": "https://www.physioboard.org.nz/sites/default/files/Physiotherapy-Board-Code-Standards-Thresholds.pdf", "filename": "physiotherapy-board-code-standards-thresholds.pdf", "format": "pdf", "section_title": "Physiotherapy Board of NZ — Advertising Standard", # Generic slicing: keep only content between start and end regex markers. # The Advertising standard appears twice in the PDF (once as full text, once # as a TOC entry); we anchor on a unique nearby string ("False, misleading or deceptive advertising # can also give rise to a breach of the Fair Trading Act 1986") to disambiguate, but # the simpler approach: slice from the FIRST occurrence of a heading-on-its-own-line # to the FIRST "Telehealth standard" that follows. # Slice between the H2 *headings* (not the bare-text TOC entries that also # appear earlier in the document). Docling preserves headings as `## Foo`, # so we require the heading prefix to disambiguate from TOC text. "slice_after": r"^## Advertising standard\s*$", "slice_until": r"^## Telehealth standard\s*$", "metadata": { "binds": "registered physiotherapists (NZ)", "issued_by": "Physiotherapy Board of New Zealand", "applies_to": "advertising of physiotherapy services; cross-references Fair Trading Act 1986, Consumer Guarantees Act 1993, ASA Codes, HPCAA s7/s8", "source_note": "extracted from the larger Physiotherapy Standards framework PDF — see source URL for full document", }, }, { "name": "Chinese Medicine Council — Advertising Standard Guidance", "url": "https://www.chinesemedicinecouncil.org.nz/common/Uploaded%20files/RegistrationsDocs/Standards%20Statement%20and%20Policies/Post%20feedback%20Advertising%20Standard%20Guidance%2023June23.pdf", "filename": "chinese-medicine-council-advertising-standard-jun-2023.pdf", "format": "pdf", "section_title": "Chinese Medicine Council of NZ — Advertising Standard Guidance", "metadata": { "binds": "registered Chinese medicine practitioners (NZ) — including acupuncturists, herbal medicine practitioners, tuina practitioners", "issued_by": "Chinese Medicine Council of New Zealand", "applies_to": "advertising of Chinese medicine services and products", "notes": "Becki flagged that AU practitioners cannot rely on traditional-use claims or 2003 WHO statement when advertising — verify whether NZ council takes the same position", }, }, { "name": "Medical Council — Statement on Advertising (BENCHMARK ONLY)", "url": "https://www.mcnz.org.nz/assets/standards/e93109ce92/Statement-on-advertising.pdf", "filename": "medical-council-statement-on-advertising.pdf", "format": "pdf", "section_title": "Medical Council of NZ — Statement on Advertising (benchmark only)", "metadata": { "binds": "registered medical practitioners (MDs) only", "benchmark-only": "true", "issued_by": "Medical Council of New Zealand", "applies_to": "advertising by registered doctors", "WARNING": "Becki: this statement is the STRICTEST in the set, particularly on testimonials. Do NOT cite as authoritative for chiropractors, osteopaths, physiotherapists, Chinese medicine practitioners, naturopaths, or other non-MD professions. Their own councils set the binding rules for them. Use this only as a comparative benchmark when explicitly asked to compare professions.", }, }, ] def fetch_guidance(source: dict) -> str: """Download a guidance PDF and convert to markdown, with binding metadata. Optional slicing: if ``slice_after`` and/or ``slice_until`` regex strings are set on the source, keep only the text BETWEEN those markers (inclusive of the start, exclusive of the end). Useful for extracting a single subdocument from a larger compendium PDF without losing the source-PDF's URL provenance. """ import re as _re print(f"\n→ {source['name']}") cache_path = SOURCES_RAW / source["filename"] if not cache_path.exists(): import urllib.request req = urllib.request.Request( source["url"], headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)"}, ) try: with urllib.request.urlopen(req) as resp: data = resp.read() cache_path.write_bytes(data) print(f" Downloaded: {cache_path.name} ({len(data) / 1024:.0f} KB)") except Exception as e: print(f" ⚠ Download failed: {e}") return "" else: print(f" Using cached: {cache_path.name}") body = extract_to_markdown(cache_path, format_hint=source.get("format")).strip() # Optional slicing — keep only the substring between markers. slice_after = source.get("slice_after") slice_until = source.get("slice_until") if slice_after or slice_until: original_size = len(body) if slice_after: m = _re.search(slice_after, body, flags=_re.MULTILINE) if m: body = body[m.start():] else: print(f" ⚠ slice_after marker {slice_after!r} not found; keeping full text") if slice_until: m = _re.search(slice_until, body, flags=_re.MULTILINE) if m: body = body[:m.start()] else: print(f" ⚠ slice_until marker {slice_until!r} not found; keeping to end") body = body.strip() print(f" ✂ Sliced: {original_size:,} → {len(body):,} chars") # Clean PDF/HTML extraction artefacts (page numbers, headers, chrome, control chars) body, _clean_stats = clean_corpus_artifacts(body) print(format_stats(_clean_stats, label=source["name"])) # Demote body headings so they nest under the source-level H2 wrapper we # add below (avoids cross-source collisions like multiple "## Introduction"). body = demote_headings(body) meta = source.get("metadata", {}) meta_lines = [f"{k}: {v}" for k, v in meta.items()] meta_block = "\n".join(meta_lines) return ( f"\n## {source['section_title']}\n\n" f"Source: {source['url']}\n\n" f"{meta_block}\n\n" f"{body}\n" ) def build(): print("Building professional_codes compilation\n") guidance_blocks = [] for src in GUIDANCE_SOURCES: block = fetch_guidance(src) if block: guidance_blocks.append(block) body = "\n\n".join(guidance_blocks) compilation = f"""# Professional Codes — NZ Healthcare Marketing Regulation Source: https://www.health.govt.nz/our-work/regulation-health-and-disability-system/health-practitioners-competence-assurance-act This compilation covers the advertising-specific standards issued by the registration boards/councils for the regulated professions in scope: chiropractors, osteopaths, physiotherapists, and Chinese medicine practitioners. The Medical Council Statement on Advertising is also included **as a benchmark only** — it does NOT bind non-MD practitioners. **Critical retrieval rule:** each council document has a `binds:` metadata line declaring which practitioners it applies to. The Medical Council statement is the strictest in the set on testimonials and should not be cited as authoritative for non-MD professions; their own councils set the binding rules. Use the binding metadata to scope answers to the practitioner type the user is asking about. {body} """ DOMAIN_FILE.write_text(compilation, encoding="utf-8") lines = compilation.count("\n") + 1 size_kb = len(compilation.encode("utf-8")) / 1024 print(f"\n✅ Wrote {DOMAIN_FILE} ({lines} lines, {size_kb:.1f} KB)") # Section-size sanity check — flags imbalanced sections that suggest a # wrong-source choice (one PDF disproportionately larger than the others). # Learning from v1 bug: 4-of-5 council docs were 13–33 KB but Physiotherapy # was 361 KB because the chosen PDF was a kitchen-sink compendium. # # Granularity note: this check counts SOURCE-LEVEL H2 sections only — the # ones we explicitly inject in `fetch_guidance` (which always have a # `Source: ...` line immediately under the heading). Docling-extracted # body content also contains H2 headings, but those are sub-sections of a # single source PDF and shouldn't be measured at this granularity. import re as _re # Match: H2 heading + blank line + Source: URL line (source-level pattern) source_section_pattern = _re.compile( r"^## (?P[^\n]+)\n\nSource: ", flags=_re.MULTILINE, ) matches = list(source_section_pattern.finditer(compilation)) section_sizes = [] for i, m in enumerate(matches): end = matches[i + 1].start() if i + 1 < len(matches) else len(compilation) section_sizes.append((end - m.start(), m.group("title"))) if section_sizes: section_sizes.sort(reverse=True) biggest = section_sizes[0][0] smallest = section_sizes[-1][0] ratio = biggest / max(smallest, 1) # Threshold tuned from observation: legitimate content variation across # focused-vs-broader council docs sits around 6–7×. The catastrophic # kitchen-sink case (e.g. v1 Physiotherapy at 26×) is the target. 8× # gives us a comfortable margin without false alarms on legitimate # imbalance (e.g. one council publishes a comprehensive code, another a # short statement). threshold = 8 print(f"\nSection size distribution (biggest/smallest ratio: {ratio:.1f}x):") for size, title in section_sizes: warn = " ⚠ disproportionate" if ratio > threshold and size == biggest else "" print(f" {size:>7,} chars {title[:70]}{warn}") if ratio > threshold: print(f"\n ⚠ One section is >{threshold}x larger than the smallest. May indicate a") print(f" wrong-source PDF (kitchen-sink compendium vs focused doc). Investigate.") if __name__ == "__main__": build()