src/mcp_services/github/repo_exporter.py

"""
repo_exporter.py – Export public GitHub repository *and* open Issues/PRs
=====================================================================
Workflow
--------
1. Mirror-clone the public repository to a local bare repo directory
   ``${out_dir}/${owner}-${repo}/repo.git``.
2. Fetch all *open* Issues & Pull-Requests via GitHub REST API (no auth
   needed for public repos, but a token can be provided to increase the rate
   limit) and serialise them as JSON under the same folder:
   • ``issues.json`` – list[Issue]
   • ``pulls.json`` – list[PullRequest]
   • ``meta.json``  – {"owner": owner, "repo": repo}

Usage (CLI)
-----------
$ python -m src.mcp_services.github.repo_exporter \
    https://github.com/octocat/Hello-World \
    --out-dir ./github_state

Optionally ``--token`` can be supplied (or env GITHUB_TOKEN) to avoid the
60-req/h anonymous limit.
"""

from __future__ import annotations

import json
import logging
import os
from dotenv import load_dotenv
import subprocess
from pathlib import Path
from tempfile import mkdtemp
from typing import Optional
from urllib.parse import urlparse

import requests

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

_API_ROOT = "https://api.github.com"
_DEFAULT_HEADERS = {
    "Accept": "application/vnd.github.v3+json",
    "User-Agent": "MCPMark/RepoExporter/1.0",
}


# ---------------------------------------------------------------------------
# Helper utilities
# ---------------------------------------------------------------------------


def _make_session(token: Optional[str] = None) -> requests.Session:
    sess = requests.Session()
    sess.headers.update(_DEFAULT_HEADERS)
    if token:
        sess.headers["Authorization"] = f"Bearer {token}"
    return sess


def _parse_repo(url: str) -> tuple[str, str]:
    parsed = urlparse(url)
    parts = parsed.path.strip("/").split("/")
    if len(parts) < 2:
        raise ValueError(f"Invalid GitHub repo URL: {url}")
    return parts[0], parts[1]


# ---------------------------------------------------------------------------
# Core export logic
# ---------------------------------------------------------------------------


def export_repository(
    source_repo_url: str,
    out_dir: str = "./github_state",
    github_token: str | None = None,
    max_issues: int | None = None,
    max_pulls: int | None = None,
) -> str:
    """Export repository code plus Issues/PRs to ``out_dir``.

    ``max_issues`` / ``max_pulls`` – when supplied, export **only** the most
    recently created *open* Issues or Pull Requests (respectively).

    Returns the absolute path of the export folder.
    """

    owner, repo = _parse_repo(source_repo_url)
    export_root = Path(out_dir).expanduser().resolve()
    repo_dir = export_root / f"{owner}-{repo}"
    repo_dir.mkdir(parents=True, exist_ok=True)

    # ------------------------------------------------------------------
    # 1. Clone repository – full or shallow *working* clone (no bare repo)
    # ------------------------------------------------------------------
    repo_path = repo_dir / "repo"
    if repo_path.exists():
        logger.info("[clone] Repository already exists, skipping clone: %s", repo_path)
    else:
        logger.info("[clone] Cloning %s/%s to %s", owner, repo, repo_path)
        env = {
            **os.environ,
            "GIT_TERMINAL_PROMPT": "0",
            "GIT_LFS_SKIP_SMUDGE": "1",
        }
        tmp_dir = mkdtemp(prefix="mcp_export_")
        try:
            # Always perform a full clone (no shallow depth limitation).
            clone_cmd = [
                "git",
                "clone",
                "--no-single-branch",
                f"https://github.com/{owner}/{repo}.git",
                tmp_dir,
            ]

            subprocess.run(clone_cmd, check=True, capture_output=True, env=env)
            subprocess.run(["mv", tmp_dir, str(repo_path)], check=True)
            logger.info("[clone] Clone completed")
        finally:
            # tmp_dir moved if success; remove if left
            if os.path.isdir(tmp_dir):
                subprocess.run(["rm", "-rf", tmp_dir])

    # ------------------------------------------------------------------
    # 2. Dump Issues & Pull Requests
    # ------------------------------------------------------------------
    sess = _make_session(github_token)

    def _paginate(url: str, state: str = "all", extra_params: dict | None = None):
        page = 1
        while True:
            params = {"state": state, "per_page": 100, "page": page}
            if extra_params:
                params.update(extra_params)
            resp = sess.get(url, params=params)
            if resp.status_code != 200:
                logger.warning("Failed to list: %s – %s", url, resp.text)
                break
            items = resp.json()
            if not items:
                break
            yield from items
            page += 1

    # --------------------------------------------------------------
    # Helper: fetch all issue comments for a given issue / PR number
    # --------------------------------------------------------------
    def _fetch_issue_comments(number: int) -> list[dict]:
        """Return a list of {user, body} comment dicts for the given issue/PR."""
        comments = []
        for c in _paginate(
            f"{_API_ROOT}/repos/{owner}/{repo}/issues/{number}/comments"
        ):
            comments.append(
                {
                    "user": c.get("user", {}).get("login", "unknown"),
                    "body": c.get("body", ""),
                }
            )
        return comments

    # --------------------------------------------------------------
    # Helper: fetch all *review* comments (code comments) for a PR
    # --------------------------------------------------------------
    def _fetch_review_comments(number: int) -> list[dict]:
        """Return a list of {user, body} review comments for the given PR."""
        comments = []
        for c in _paginate(f"{_API_ROOT}/repos/{owner}/{repo}/pulls/{number}/comments"):
            comments.append(
                {
                    "user": c.get("user", {}).get("login", "unknown"),
                    "body": c.get("body", ""),
                }
            )
        return comments

    # Issues (non-PR)
    issues = []
    # If max_issues is 0, skip fetching issues entirely
    if max_issues == 0:
        logger.info("[export] Skipping issues (max_issues=0)")
    else:
        for itm in _paginate(
            f"{_API_ROOT}/repos/{owner}/{repo}/issues",
            extra_params={"sort": "created", "direction": "desc"},
        ):
            if "pull_request" in itm:
                continue
            issues.append(
                {
                    "title": itm.get("title"),
                    "body": itm.get("body", ""),
                    "labels": [lbl.get("name") for lbl in itm.get("labels", [])],
                    "state": itm.get("state", "open"),  # Store issue state
                    "number": itm.get("number"),  # Store issue number for reference
                    "comments": _fetch_issue_comments(itm.get("number")),
                }
            )

            if max_issues is not None and len(issues) >= max_issues:
                break
    (repo_dir / "issues.json").write_text(json.dumps(issues, indent=2))
    logger.info("[export] Saved %d issues", len(issues))

    # Pull requests – include *all* PRs including those from forks
    pulls = []
    pr_head_refs: set[str] = set()
    fork_pr_branches: dict[str, dict] = {}  # Maps PR branch names to fork info

    # If max_pulls is 0, skip fetching pull requests entirely
    if max_pulls == 0:
        logger.info("[export] Skipping pull requests (max_pulls=0)")
    else:
        for pr in _paginate(
            f"{_API_ROOT}/repos/{owner}/{repo}/pulls",
            state="open",
            extra_params={"sort": "created", "direction": "desc"},
        ):
            pr_number = pr.get("number")
            head = pr.get("head", {})
            if head is None:
                logger.warning("PR #%s has no head (deleted fork), skipping", pr_number)
                continue  # skip PRs with missing head (deleted fork)

            head_repo = head.get("repo")
            head_ref = head.get("ref")
            head_sha = head.get("sha")

            if head_repo is None:
                logger.warning("PR #%s source repo was deleted, skipping", pr_number)
                continue  # skip PRs where source repo was deleted

            head_repo_full = head_repo.get("full_name")
            is_from_fork = head_repo_full != f"{owner}/{repo}"

            # Create PR data with fork information
            pr_data = {
                "number": pr_number,
                "title": pr.get("title"),
                "body": pr.get("body", ""),
                "head": head_ref,
                "base": pr.get("base", {}).get("ref"),
                "is_from_fork": is_from_fork,
            }

            if is_from_fork:
                # Store additional metadata for forked PRs
                pr_data["fork_owner"] = head_repo.get("owner", {}).get("login")
                pr_data["fork_repo"] = head_repo.get("name")
                pr_data["head_sha"] = head_sha

                # Create a unique branch name for this forked PR
                fork_branch_name = f"pr/{pr_number}-{pr_data['fork_owner']}-{head_ref}"
                pr_data["local_branch"] = fork_branch_name

                fork_pr_branches[fork_branch_name] = {
                    "clone_url": head_repo.get("clone_url"),
                    "ref": head_ref,
                    "sha": head_sha,
                    "pr_number": pr_number,
                }
            else:
                # For non-fork PRs, keep the original branch reference
                pr_head_refs.add(head_ref)

            # Attach comments
            pr_data["comments"] = _fetch_issue_comments(pr_number)
            pr_data["review_comments"] = _fetch_review_comments(pr_number)

            pulls.append(pr_data)

            if max_pulls is not None and len(pulls) >= max_pulls:
                break
    (repo_dir / "pulls.json").write_text(json.dumps(pulls, indent=2))
    logger.info("[export] Saved %d pull requests", len(pulls))

    # Get default branch info first (needed for fetching)
    sess = _make_session(github_token)
    try:
        repo_info = sess.get(f"{_API_ROOT}/repos/{owner}/{repo}")
        default_branch = repo_info.json().get("default_branch", "main")
    except Exception:
        default_branch = "main"

    # Fetch branches from non-fork PRs (branches from the same repository)
    non_fork_branches = list(pr_head_refs)  # These are branches from the same repo
    # Always include the default branch in the branches to fetch
    if default_branch not in non_fork_branches:
        non_fork_branches.append(default_branch)
        pr_head_refs.add(default_branch)

    if non_fork_branches:
        logger.info(
            "[fetch] Fetching %d branches from same repository (including default branch '%s')",
            len(non_fork_branches),
            default_branch,
        )
        try:
            # Fetch all remote branches to ensure we have the PR branches
            subprocess.run(
                ["git", "-C", str(repo_path), "fetch", "origin", "--no-tags"],
                check=True,
                capture_output=True,
            )

            # Create local branches for each PR branch
            for branch in non_fork_branches:
                try:
                    # Create local branch tracking the remote branch
                    subprocess.run(
                        [
                            "git",
                            "-C",
                            str(repo_path),
                            "branch",
                            "--track",
                            branch,
                            f"origin/{branch}",
                        ],
                        check=False,
                        capture_output=True,
                    )  # check=False because branch might already exist
                    logger.info("[fetch] Created local branch %s", branch)
                except subprocess.CalledProcessError:
                    # Branch might already exist, which is fine
                    pass

        except subprocess.CalledProcessError as e:
            logger.warning(
                "[fetch] Failed to fetch branches from origin: %s",
                e.stderr.decode(errors="ignore") if e.stderr else str(e),
            )

    # Fetch branches from forks for PRs
    if fork_pr_branches:
        logger.info(
            "[fetch] Fetching branches from %d forked PRs", len(fork_pr_branches)
        )

        for branch_name, fork_info in fork_pr_branches.items():
            try:
                logger.info(
                    "[fetch] Fetching branch %s from fork %s",
                    fork_info["ref"],
                    fork_info["clone_url"],
                )

                # Add fork as remote and fetch the specific branch
                remote_name = f"fork-pr-{fork_info['pr_number']}"

                # Add remote
                subprocess.run(
                    [
                        "git",
                        "-C",
                        str(repo_path),
                        "remote",
                        "add",
                        remote_name,
                        fork_info["clone_url"],
                    ],
                    check=True,
                    capture_output=True,
                )

                # Fetch the specific branch from the fork
                subprocess.run(
                    [
                        "git",
                        "-C",
                        str(repo_path),
                        "fetch",
                        remote_name,
                        f"{fork_info['ref']}:refs/heads/{branch_name}",
                    ],
                    check=True,
                    capture_output=True,
                )

                # Remove the remote after fetching
                subprocess.run(
                    ["git", "-C", str(repo_path), "remote", "remove", remote_name],
                    check=True,
                    capture_output=True,
                )

                # Add the fork branch to pr_head_refs so it gets pushed
                pr_head_refs.add(branch_name)

                logger.info("[fetch] Successfully fetched branch %s", branch_name)

            except subprocess.CalledProcessError as e:
                logger.warning(
                    "[fetch] Failed to fetch branch from fork PR #%s: %s",
                    fork_info["pr_number"],
                    e.stderr.decode(errors="ignore") if e.stderr else str(e),
                )
            except Exception as e:
                logger.warning(
                    "[fetch] Unexpected error fetching fork PR #%s: %s",
                    fork_info["pr_number"],
                    str(e),
                )

    meta = {
        "owner": owner,
        "repo": repo,
        "default_branch": default_branch,
        "pr_head_refs": sorted(pr_head_refs),
    }
    (repo_dir / "meta.json").write_text(json.dumps(meta, indent=2))

    logger.info("[done] Export finished – data stored at %s", repo_dir)
    return str(repo_dir)


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
if __name__ == "__main__":
    import argparse

    load_dotenv(".mcp_env")

    parser = argparse.ArgumentParser(
        description="Export public GitHub repository with Issues/PRs"
    )
    parser.add_argument(
        "--source_repo_url", required=True, help="HTTPS URL of the public repository"
    )
    parser.add_argument(
        "--out-dir", default="./github_state", help="Output directory root"
    )
    parser.add_argument(
        "--max-issues",
        type=int,
        default=20,
        help="Export only the latest N issues (optional)",
    )
    parser.add_argument(
        "--max-pulls",
        type=int,
        default=5,
        help="Export only the latest N pull requests (optional)",
    )
    args = parser.parse_args()

    token = os.getenv("GITHUB_TOKEN")

    export_repository(
        args.source_repo_url, args.out_dir, token, args.max_issues, args.max_pulls
    )