src/regx/cli.py

from __future__ import annotations
import json, os, sys, click
from .fetch import fetch_url, manifest_to_source
from .parse_html import parse_html
from .parse_pdf import parse_pdf
from .models import SourceMeta, Node, Chunk
from .chunker import make_chunks
from .concepts import auto_concepts

@click.group()
def cli(): ...

@cli.command("fetch-url")
@click.argument("url")
@click.option("--out", default="data", help="Output directory for artifact + manifest")
def cmd_fetch(url, out):
    meta_path = fetch_url(url, out)
    click.echo(meta_path)

@cli.command("parse-structured")
@click.argument("manifest_path")
@click.option("--out", required=True, help="Output JSON file for Node tree")
@click.option("--title", default=None)
@click.option("--jurisdiction", default="OTHER")
def cmd_parse(manifest_path, out, title, jurisdiction):
    source = manifest_to_source(manifest_path, title=title, jurisdiction=jurisdiction)
    media = source.media_type
    if "pdf" in media:
        root = parse_pdf(manifest_path, source)
    elif "html" in media or "xml" in media or "xhtml" in media or "text/plain" in media:
        root = parse_html(manifest_path, source)
    else:
        raise click.ClickException(f"Unsupported media_type: {media}")
    with open(out, "w") as f:
        json.dump(root.model_dump(mode="json"), f, indent=2, ensure_ascii=False)
    click.echo(out)

@cli.command("chunk")
@click.argument("nodes_json")
@click.option("--out", required=True, help="Output JSONL with chunks")
@click.option("--max-tokens", default=1024, type=int)
@click.option("--min-tokens", default=200, type=int)
def cmd_chunk(nodes_json, out, max_tokens, min_tokens):
    with open(nodes_json) as f:
        root = Node.model_validate_json(f.read())
    # SourceMeta is not serialized in nodes.json; reconstruct from a sibling manifest if present
    # Fallback dummy SourceMeta so chunks remain valid
    dummy = SourceMeta(url="https://example.invalid", media_type="text/html")
    os.makedirs(os.path.dirname(out) or ".", exist_ok=True)
    with open(out, "w") as fo:
        for ch in make_chunks(root, dummy, max_tokens=max_tokens, min_tokens=min_tokens):
            fo.write(json.dumps(ch.model_dump(mode="json"), ensure_ascii=False) + "\n")
    click.echo(out)

@cli.command("map-concepts")
@click.argument("chunks_jsonl")
@click.option("--out", required=True, help="Output JSONL with {chunk_id, concepts}")
def cmd_map(chunks_jsonl, out):
    with open(chunks_jsonl) as f, open(out, "w") as fo:
        for line in f:
            obj = json.loads(line)
            text = obj.get("text","")
            concepts = auto_concepts(text)
            obj["concepts"] = concepts
            fo.write(json.dumps(obj, ensure_ascii=False) + "\n")
    click.echo(out)

def main():
    cli()

if __name__ == "__main__":
    main()