fetch_licenses.py

#!/usr/bin/env python3
"""Fetch the Gemma licence texts that must ship alongside redistributed weights.

Apache 2.0 has a canonical plaintext URL and is downloaded verbatim. The two Gemma
documents only exist as HTML pages, so their article body is extracted to text.

The extracted files are a convenience, not an authority: diff them against the live
pages before publishing a repo that relies on them. Re-run whenever Google updates
the terms.

Usage:  python fetch_licenses.py
"""

import re
import sys
import urllib.request

try:
    from bs4 import BeautifulSoup, NavigableString, Tag
except ImportError:
    sys.exit("需要 beautifulsoup4：pip install beautifulsoup4")

PLAINTEXT = {
    "LICENSE-apache-2.0.txt": "https://www.apache.org/licenses/LICENSE-2.0.txt",
}

# Both shipped models are Apache 2.0, so nothing here needs scraping today. Keep the entries
# commented rather than deleting the machinery: re-adding any Gemma 1/1.1/2/3/3n model brings
# these obligations straight back, and the extraction is fiddly enough to be worth preserving.
HTML_PAGES: dict[str, str] = {
    # "LICENSE-gemma-terms.txt": "https://ai.google.dev/gemma/terms",
    # "PROHIBITED_USE_POLICY.txt": "https://ai.google.dev/gemma/prohibited_use_policy",
}

BLOCK_TAGS = {"p", "li", "h1", "h2", "h3", "h4", "h5", "h6", "tr", "div", "section"}
HEADING_TAGS = {"h1", "h2", "h3", "h4", "h5", "h6"}


def get(url: str) -> bytes:
    request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
    with urllib.request.urlopen(request, timeout=30) as response:
        return response.read()


def render(node, out, depth=0):
    """Walk the article tree emitting one line per block element."""
    if isinstance(node, NavigableString):
        text = str(node).strip()
        if text:
            out.append(("inline", text))
        return
    if not isinstance(node, Tag):
        return
    if node.name in {"script", "style", "nav", "button"}:
        return

    if node.name in BLOCK_TAGS:
        text = " ".join(node.get_text(" ", strip=True).split())
        if text:
            # Only emit leaf-ish blocks; container divs would duplicate their children.
            has_block_child = any(
                isinstance(c, Tag) and c.name in BLOCK_TAGS for c in node.children
            )
            if not has_block_child:
                prefix = "- " if node.name == "li" else ""
                kind = "heading" if node.name in HEADING_TAGS else "block"
                out.append((kind, prefix + text))
                return
    for child in node.children:
        render(child, out, depth + 1)


def tidy(text: str) -> str:
    """Undo spacing artefacts left by inline tags around defined terms."""
    text = re.sub(r'"\s+(.*?)\s+"', r'"\1"', text)
    return re.sub(r"\s+([.,;:])", r"\1", text)


def html_to_text(html: bytes) -> str:
    soup = BeautifulSoup(html, "html.parser")
    article = soup.find("article") or soup.body
    if article is None:
        raise SystemExit("找不到文章內容，頁面結構可能已改變")

    out = []
    render(article, out)
    # Drop the site chrome (release banner, breadcrumbs) preceding the document title.
    first_heading = next((i for i, (kind, _) in enumerate(out) if kind == "heading"), 0)
    out = [(kind, tidy(text)) for kind, text in out[first_heading:]]

    lines = []
    for kind, text in out:
        if kind == "heading":
            lines.append("")
            lines.append(text)
            lines.append("=" * len(text))
        else:
            lines.append(text)
        lines.append("")
    # Collapse runs of blank lines.
    result, blank = [], False
    for line in lines:
        if line.strip() == "":
            if not blank:
                result.append("")
            blank = True
        else:
            result.append(line)
            blank = False
    return "\n".join(result).strip() + "\n"


def main():
    for name, url in PLAINTEXT.items():
        print(f"下載 {name} <- {url}")
        with open(name, "wb") as handle:
            handle.write(get(url))

    for name, url in HTML_PAGES.items():
        print(f"擷取 {name} <- {url}")
        text = html_to_text(get(url))
        with open(name, "w", encoding="utf-8") as handle:
            handle.write(text)
        print(f"  {len(text.splitlines())} 行，請人工核對")

    print("\n完成。上傳前務必與官方頁面對照一次擷取出來的兩份 Gemma 文件。")


if __name__ == "__main__":
    main()