Add Python package entry point

README.md

@@ -27,6 +27,26 @@ The app includes two modes:
 
 The model never edits code directly. It only proposes comments, and the app rejects any annotated file whose semantic AST changes.
 
+## CLI Usage
+
+The same annotation pipeline is available from the command line:
+
+```bash
+pip install -e .
+between-the-lines path/to/file.py --model base --output annotated.py --summary
+btl path/to/file.py --check --summary
+```
+
+You can also run it without installing a console command:
+
+```bash
+python -m btl.cli path/to/file.py --model base --output annotated.py --summary
+python -m btl.cli path/to/file.py --model tuned --in-place
+python -m btl.cli path/to/file.py --check --summary
+```
+
+`--model base` uses the richer Mellum2 GGUF path. `--model tuned` uses the LoRA adapter for shorter comments. Both modes run the AST validation before writing output.
+
 ## Hackathon Fit
 
 - **Track:** Backyard AI

app.py

@@ -1,12 +1,11 @@
-import ast
 import html
-from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
 import gradio as gr
 
-from btl.model import ModelUnavailableError, generate_comment, generate_comment_with_llm, load_llm
+from btl.annotate import annotate_python, collect_blocks, parse_python
+from btl.model import ModelUnavailableError
 
 
 CSS = """
@@ -156,130 +155,18 @@ code {
 """
 
 
-@dataclass(frozen=True)
-class BlockInfo:
-    kind: str
-    name: str
-    lineno: int
-    source: str
-
-
-def _parse_python(source: str) -> ast.Module:
-    return ast.parse(source)
-
-
-def _strip_docstrings(node: ast.AST) -> ast.AST:
-    copied = ast.fix_missing_locations(ast.parse(ast.unparse(node)))
-    for child in ast.walk(copied):
-        if isinstance(child, (ast.Module, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
-            if (
-                child.body
-                and isinstance(child.body[0], ast.Expr)
-                and isinstance(child.body[0].value, ast.Constant)
-                and isinstance(child.body[0].value.value, str)
-            ):
-                child.body = child.body[1:]
-    return copied
-
-
-def _semantic_ast_dump(source: str) -> str:
-    tree = _parse_python(source)
-    stripped = _strip_docstrings(tree)
-    return ast.dump(stripped, include_attributes=False)
-
-
-def _collect_blocks(tree: ast.Module, source: str) -> list[BlockInfo]:
-    blocks: list[BlockInfo] = []
-    for node in ast.walk(tree):
-        if isinstance(node, ast.ClassDef):
-            block_source = ast.get_source_segment(source, node) or ast.unparse(node)
-            blocks.append(BlockInfo("class", node.name, node.lineno, block_source))
-        elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
-            kind = "async function" if isinstance(node, ast.AsyncFunctionDef) else "function"
-            block_source = ast.get_source_segment(source, node) or ast.unparse(node)
-            blocks.append(BlockInfo(kind, node.name, node.lineno, block_source))
-    return sorted(blocks, key=lambda item: item.lineno)
-
-
-def _placeholder_summary(blocks: list[BlockInfo]) -> str:
-    if not blocks:
-        return "This Python file has no classes or functions to annotate yet."
-    names = ", ".join(f"{block.kind} `{block.name}`" for block in blocks[:8])
-    overflow = "" if len(blocks) <= 8 else f", plus {len(blocks) - 8} more"
-    return f"This file defines {names}{overflow}. Model-generated summaries will replace this deterministic placeholder."
-
-
-def _insert_comments(source: str, comments: dict[int, str]) -> str:
-    lines = source.splitlines()
-    inserts: dict[int, list[str]] = {}
-    for lineno, comment_text in comments.items():
-        indent = len(lines[lineno - 1]) - len(lines[lineno - 1].lstrip())
-        inserts.setdefault(lineno - 1, []).append(" " * indent + comment_text)
-
-    annotated: list[str] = []
-    for index, line in enumerate(lines):
-        annotated.extend(inserts.get(index, []))
-        annotated.append(line)
-    return "\n".join(annotated) + ("\n" if source.endswith("\n") else "")
-
-
 MODEL_LABELS = {
     "Base Mellum2 (richer)": "base",
     "Fine-tuned LoRA (concise)": "tuned",
 }
 
 
-def _generate_block_comments(blocks: list[BlockInfo], model_label: str) -> tuple[dict[int, str], list[str]]:
-    comments: dict[int, str] = {}
-    notes: list[str] = []
-    model_variant = MODEL_LABELS.get(model_label, "base")
-    llm = load_llm() if model_variant == "base" else None
-    for block in blocks:
-        try:
-            if model_variant == "base":
-                comments[block.lineno] = generate_comment_with_llm(llm, block.kind, block.name, block.source)
-            else:
-                comments[block.lineno] = generate_comment(block.kind, block.name, block.source, variant="tuned")
-        except Exception as exc:
-            comments[block.lineno] = f"# TODO(model): Could not explain {block.kind} `{block.name}`."
-            notes.append(f"{block.name}: model generation failed ({type(exc).__name__}).")
-    return comments, notes
-
-
 def annotate_code(source: str, model_label: str) -> tuple[str, str, str]:
-    source = source.strip("\ufeff")
-    if not source.strip():
-        return "", "", "Paste a Python file to annotate."
-
-    try:
-        original_tree = _parse_python(source)
-    except SyntaxError as exc:
-        return "", "", f"Syntax error on line {exc.lineno}: {html.escape(exc.msg)}"
-
-    blocks = _collect_blocks(original_tree, source)
-    if not blocks:
-        return _placeholder_summary(blocks), source, "Parsed successfully. No classes or functions to annotate."
-
     try:
-        comments, generation_notes = _generate_block_comments(blocks, model_label)
+        result = annotate_python(source, MODEL_LABELS.get(model_label, "base"))
     except ModelUnavailableError as exc:
-        return _placeholder_summary(blocks), source, f"Model unavailable: {html.escape(str(exc))}"
-
-    annotated = _insert_comments(source, comments)
-
-    try:
-        same_ast = _semantic_ast_dump(source) == _semantic_ast_dump(annotated)
-    except SyntaxError as exc:
-        return "", annotated, f"Generated annotation failed to parse on line {exc.lineno}: {html.escape(exc.msg)}"
-
-    status = (
-        f"Validated: parsed {len(blocks)} block(s), inserted {model_label} comments, semantic AST unchanged."
-        if same_ast
-        else "Rejected: annotation changed the semantic AST."
-    )
-    if generation_notes:
-        status += "\n" + "\n".join(generation_notes)
-    return _placeholder_summary(blocks), annotated if same_ast else source, status
+        return "", source, f"Model unavailable: {html.escape(str(exc))}"
+    return result.summary, result.annotated_source, result.status
 
 
 def load_uploaded_python(file_obj: Any) -> tuple[str, str]:
@@ -298,11 +185,11 @@ def load_uploaded_python(file_obj: Any) -> tuple[str, str]:
         return "", f"Could not read `{path.name}`: {exc}"
 
     try:
-        tree = _parse_python(source)
+        tree = parse_python(source)
     except SyntaxError as exc:
         return source, f"Loaded `{path.name}`, but parsing failed on line {exc.lineno}: {html.escape(exc.msg)}"
 
-    blocks = _collect_blocks(tree, source)
+    blocks = collect_blocks(tree, source)
     return source, f"Loaded `{path.name}`. Parsed {len(blocks)} class/function block(s)."
 
 

btl/annotate.py

@@ -0,0 +1,152 @@
+import ast
+from dataclasses import dataclass
+from typing import Literal
+
+from .model import ModelUnavailableError, generate_comment, generate_comment_with_llm, load_llm
+
+
+ModelChoice = Literal["base", "tuned"]
+
+
+@dataclass(frozen=True)
+class BlockInfo:
+    kind: str
+    name: str
+    lineno: int
+    source: str
+
+
+@dataclass(frozen=True)
+class AnnotationResult:
+    summary: str
+    annotated_source: str
+    status: str
+    valid: bool
+    block_count: int
+    notes: tuple[str, ...] = ()
+
+
+def parse_python(source: str) -> ast.Module:
+    return ast.parse(source)
+
+
+def strip_docstrings(node: ast.AST) -> ast.AST:
+    copied = ast.fix_missing_locations(ast.parse(ast.unparse(node)))
+    for child in ast.walk(copied):
+        if isinstance(child, (ast.Module, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+            if (
+                child.body
+                and isinstance(child.body[0], ast.Expr)
+                and isinstance(child.body[0].value, ast.Constant)
+                and isinstance(child.body[0].value.value, str)
+            ):
+                child.body = child.body[1:]
+    return copied
+
+
+def semantic_ast_dump(source: str) -> str:
+    tree = parse_python(source)
+    stripped = strip_docstrings(tree)
+    return ast.dump(stripped, include_attributes=False)
+
+
+def collect_blocks(tree: ast.Module, source: str) -> list[BlockInfo]:
+    blocks: list[BlockInfo] = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ClassDef):
+            block_source = ast.get_source_segment(source, node) or ast.unparse(node)
+            blocks.append(BlockInfo("class", node.name, node.lineno, block_source))
+        elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            kind = "async function" if isinstance(node, ast.AsyncFunctionDef) else "function"
+            block_source = ast.get_source_segment(source, node) or ast.unparse(node)
+            blocks.append(BlockInfo(kind, node.name, node.lineno, block_source))
+    return sorted(blocks, key=lambda item: item.lineno)
+
+
+def summarize_blocks(blocks: list[BlockInfo]) -> str:
+    if not blocks:
+        return "This Python file has no classes or functions to annotate yet."
+    names = ", ".join(f"{block.kind} `{block.name}`" for block in blocks[:8])
+    overflow = "" if len(blocks) <= 8 else f", plus {len(blocks) - 8} more"
+    return f"This file defines {names}{overflow}."
+
+
+def insert_comments(source: str, comments: dict[int, str]) -> str:
+    lines = source.splitlines()
+    inserts: dict[int, list[str]] = {}
+    for lineno, comment_text in comments.items():
+        indent = len(lines[lineno - 1]) - len(lines[lineno - 1].lstrip())
+        inserts.setdefault(lineno - 1, []).append(" " * indent + comment_text)
+
+    annotated: list[str] = []
+    for index, line in enumerate(lines):
+        annotated.extend(inserts.get(index, []))
+        annotated.append(line)
+    return "\n".join(annotated) + ("\n" if source.endswith("\n") else "")
+
+
+def generate_block_comments(blocks: list[BlockInfo], model_choice: ModelChoice = "base") -> tuple[dict[int, str], list[str]]:
+    comments: dict[int, str] = {}
+    notes: list[str] = []
+    llm = load_llm() if model_choice == "base" else None
+
+    for block in blocks:
+        try:
+            if model_choice == "base":
+                comments[block.lineno] = generate_comment_with_llm(llm, block.kind, block.name, block.source)
+            else:
+                comments[block.lineno] = generate_comment(block.kind, block.name, block.source, variant="tuned")
+        except ModelUnavailableError:
+            raise
+        except Exception as exc:
+            comments[block.lineno] = f"# TODO(model): Could not explain {block.kind} `{block.name}`."
+            notes.append(f"{block.name}: model generation failed ({type(exc).__name__}).")
+    return comments, notes
+
+
+def annotate_python(source: str, model_choice: ModelChoice = "base") -> AnnotationResult:
+    source = source.strip("\ufeff")
+    if not source.strip():
+        return AnnotationResult("", "", "Paste a Python file to annotate.", False, 0)
+
+    try:
+        original_tree = parse_python(source)
+    except SyntaxError as exc:
+        return AnnotationResult("", "", f"Syntax error on line {exc.lineno}: {exc.msg}", False, 0)
+
+    blocks = collect_blocks(original_tree, source)
+    summary = summarize_blocks(blocks)
+    if not blocks:
+        return AnnotationResult(summary, source, "Parsed successfully. No classes or functions to annotate.", True, 0)
+
+    comments, generation_notes = generate_block_comments(blocks, model_choice)
+    annotated = insert_comments(source, comments)
+
+    try:
+        same_ast = semantic_ast_dump(source) == semantic_ast_dump(annotated)
+    except SyntaxError as exc:
+        return AnnotationResult(
+            "",
+            annotated,
+            f"Generated annotation failed to parse on line {exc.lineno}: {exc.msg}",
+            False,
+            len(blocks),
+            tuple(generation_notes),
+        )
+
+    status = (
+        f"Validated: parsed {len(blocks)} block(s), inserted {model_choice} model comments, semantic AST unchanged."
+        if same_ast
+        else "Rejected: annotation changed the semantic AST."
+    )
+    if generation_notes:
+        status += "\n" + "\n".join(generation_notes)
+
+    return AnnotationResult(
+        summary,
+        annotated if same_ast else source,
+        status,
+        same_ast,
+        len(blocks),
+        tuple(generation_notes),
+    )

btl/cli.py

@@ -0,0 +1,96 @@
+import argparse
+import sys
+from pathlib import Path
+
+from .annotate import annotate_python
+from .model import ModelUnavailableError
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="between-the-lines",
+        description="Annotate a Python file with model-generated comments and verify the semantic AST is unchanged.",
+    )
+    parser.add_argument("input", type=Path, help="Python file to annotate.")
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        help="Write annotated Python to this file. Defaults to stdout.",
+    )
+    parser.add_argument(
+        "--model",
+        choices=["base", "tuned"],
+        default="base",
+        help="Comment model to use. Defaults to base.",
+    )
+    parser.add_argument(
+        "--in-place",
+        action="store_true",
+        help="Overwrite the input file with the annotated version after validation.",
+    )
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="Parse and validate only; do not write annotated code.",
+    )
+    parser.add_argument(
+        "--summary",
+        action="store_true",
+        help="Print the file summary and validation status to stderr.",
+    )
+    return parser.parse_args()
+
+
+def read_source(path: Path) -> str:
+    try:
+        return path.read_text(encoding="utf-8")
+    except UnicodeDecodeError:
+        return path.read_text(encoding="utf-8-sig")
+
+
+def main() -> int:
+    args = parse_args()
+    if args.output and args.in_place:
+        print("error: use either --output or --in-place, not both", file=sys.stderr)
+        return 2
+
+    try:
+        source = read_source(args.input)
+    except OSError as exc:
+        print(f"error: could not read {args.input}: {exc}", file=sys.stderr)
+        return 2
+    except UnicodeDecodeError as exc:
+        print(f"error: could not decode {args.input} as UTF-8 Python text: {exc}", file=sys.stderr)
+        return 2
+
+    try:
+        result = annotate_python(source, args.model)
+    except ModelUnavailableError as exc:
+        print(f"error: model unavailable: {exc}", file=sys.stderr)
+        return 1
+    if args.summary or args.check:
+        if result.summary:
+            print(result.summary, file=sys.stderr)
+        print(result.status, file=sys.stderr)
+
+    if not result.valid:
+        return 1
+
+    if args.check:
+        return 0
+
+    if args.in_place:
+        args.input.write_text(result.annotated_source, encoding="utf-8")
+        return 0
+
+    if args.output:
+        args.output.write_text(result.annotated_source, encoding="utf-8")
+        return 0
+
+    print(result.annotated_source, end="")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyproject.toml

@@ -0,0 +1,29 @@
+[build-system]
+requires = ["setuptools>=69", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "between-the-lines"
+version = "0.1.0"
+description = "Annotate Python files with small-model comments and AST validation."
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "accelerate",
+    "bitsandbytes",
+    "gradio==5.50.0",
+    "huggingface_hub",
+    "llama-cpp-python",
+    "peft",
+    "safetensors",
+    "torch",
+    "transformers @ git+https://github.com/huggingface/transformers.git",
+    "truststore",
+]
+
+[project.scripts]
+between-the-lines = "btl.cli:main"
+btl = "btl.cli:main"
+
+[tool.setuptools.packages.find]
+include = ["btl*"]