.github/workflows/huggingface-sync.yml

name: Sync Hugging Face

on:
  push:
    branches:
      - main
  workflow_dispatch:

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
  HF_TOKEN: ${{ secrets.HF_TOKEN }}

permissions:
  contents: read

concurrency:
  group: "huggingface-sync-main"
  cancel-in-progress: true

jobs:
  sync:
    name: Sync model repo
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          fetch-depth: 0
          lfs: false

      - name: Skip when HF_TOKEN is not configured
        if: ${{ env.HF_TOKEN == '' }}
        run: |
          echo "::notice title=Hugging Face sync skipped::Set the HF_TOKEN repository secret to publish Stevesolun/ctx automatically."

      - name: Classify sync scope
        id: scope
        if: ${{ env.HF_TOKEN != '' }}
        shell: bash
        env:
          BEFORE_SHA: ${{ github.event.before }}
          EVENT_NAME: ${{ github.event_name }}
          HEAD_SHA: ${{ github.sha }}
        run: |
          set -euo pipefail
          if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then
            echo "sync_mode=full" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          BASE="$BEFORE_SHA"
          if [[ -z "$BASE" || "$BASE" =~ ^0+$ ]] || ! git cat-file -e "$BASE^{commit}" 2>/dev/null; then
            BASE="$(git rev-parse "$HEAD_SHA^")"
          fi
          git diff --name-only "$BASE" "$HEAD_SHA" > changed-files.txt
          python - <<'PY' >> "$GITHUB_OUTPUT"
          from pathlib import Path

          files = [
              line.strip().replace("\\", "/")
              for line in Path("changed-files.txt").read_text(encoding="utf-8").splitlines()
              if line.strip()
          ]
          card_only_prefixes = (".github/", "docs/", "src/tests/")
          card_only_files = {"README.md", "CHANGELOG.md"}
          card_only = bool(files) and all(
              path in card_only_files or path.startswith(card_only_prefixes)
              for path in files
          )
          print(f"sync_mode={'card' if card_only else 'full'}")
          PY

      - name: Hydrate required graph artifacts from release assets
        if: ${{ env.HF_TOKEN != '' && steps.scope.outputs.sync_mode == 'full' }}
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          echo "Resolving graph artifacts from matching release assets to avoid Git LFS bandwidth."
          python - <<'PY'
          import hashlib
          import json
          import os
          from pathlib import Path
          import subprocess
          import time
          import urllib.request

          repo = os.environ["GITHUB_REPOSITORY"]
          release_asset_wait_seconds = 300
          release_asset_poll_seconds = 10
          expected_graph_assets = {
              "graph/wiki-graph.tar.gz": {
                  "sha256": "d051d4f21208abe3e73975a9c30650150138d88919c27979d725be936a6ea10a",
                  "size": 296113770,
              },
              "graph/wiki-graph-runtime.tar.gz": {
                  "sha256": "993fc08377fdb09edcff4414c59b10fc121189b4a161bf796e3f8f6600907bb1",
                  "size": 122141091,
              },
              "graph/skills-sh-catalog.json.gz": {
                  "sha256": "d1bb927e93d65ef86cdbc6d68c7f57511d64c8f13026e8193287a9c4ca6f5815",
                  "size": 10763626,
              },
          }

          def load_releases() -> list[dict]:
              return json.loads(subprocess.check_output(
                  ["gh", "api", f"repos/{repo}/releases?per_page=50"],
                  text=True,
              ))

          def hydrate_from_release(path_name: str, hydrated_min_size: int) -> None:
              artifact = Path(path_name)
              fallback = expected_graph_assets[path_name]
              expected_oid = fallback["sha256"]
              expected_size = int(fallback["size"])
              if artifact.exists():
                  pointer = artifact.read_text(encoding="utf-8", errors="replace")
                  for line in pointer.splitlines():
                      if line.startswith("oid sha256:"):
                          expected_oid = line.split(":", 1)[1].strip()
                      elif line.startswith("size "):
                          expected_size = int(line.split(" ", 1)[1].strip())
                  if not pointer.startswith("version https://git-lfs.github.com/spec/v1") and artifact.stat().st_size > hydrated_min_size:
                      print(f"{artifact} is already hydrated")
                      return

              deadline = time.monotonic() + release_asset_wait_seconds
              while True:
                  candidates = []
                  for release in load_releases():
                      tag_name = str(release.get("tag_name") or "")
                      is_graph_cache = tag_name.startswith("graph-artifacts-")
                      if release.get("draft") or (
                          release.get("prerelease") and not is_graph_cache
                      ):
                          continue
                      for asset in release.get("assets", []):
                          if asset.get("name") != artifact.name:
                              continue
                          digest = str(asset.get("digest") or "")
                          size = int(asset.get("size") or 0)
                          if size != expected_size:
                              continue
                          if digest and digest != f"sha256:{expected_oid}":
                              continue
                          candidates.append((tag_name, asset))

                  if candidates:
                      break
                  if time.monotonic() >= deadline:
                      raise SystemExit(
                          f"No release asset matches {path_name} "
                          f"sha256:{expected_oid} size:{expected_size}"
                      )
                  print(
                      f"Waiting for matching release asset {artifact.name} "
                      f"sha256:{expected_oid} size:{expected_size}"
                  )
                  time.sleep(release_asset_poll_seconds)

              source_tag, asset = candidates[0]
              tmp = artifact.with_name(f"{artifact.name}.download")
              sha = hashlib.sha256()
              total = 0
              with urllib.request.urlopen(asset["browser_download_url"], timeout=300) as resp:  # noqa: S310
                  with tmp.open("wb") as fh:
                      while True:
                          chunk = resp.read(1024 * 1024)
                          if not chunk:
                              break
                          sha.update(chunk)
                          total += len(chunk)
                          fh.write(chunk)
              actual_oid = sha.hexdigest()
              if actual_oid != expected_oid or total != expected_size:
                  tmp.unlink(missing_ok=True)
                  raise SystemExit(
                      f"Downloaded {path_name} does not match expected artifact: "
                      f"sha256:{actual_oid} size:{total}"
                  )
              tmp.replace(artifact)
              print(
                  f"Hydrated {path_name} from {source_tag} release asset "
                  f"sha256:{actual_oid} size:{total}"
              )

          hydrate_from_release("graph/wiki-graph.tar.gz", 100_000_000)
          hydrate_from_release("graph/wiki-graph-runtime.tar.gz", 10_000_000)
          hydrate_from_release("graph/skills-sh-catalog.json.gz", 1_000_000)
          PY

      - name: Set up Python
        if: ${{ env.HF_TOKEN != '' }}
        uses: actions/setup-python@v6
        with:
          python-version: "3.11"
          cache: "pip"
          cache-dependency-path: pyproject.toml

      - name: Install sync dependencies
        if: ${{ env.HF_TOKEN != '' }}
        run: |
          python -m pip install --upgrade pip
          python -m pip install -e ".[dev]" huggingface_hub

      - name: Sync Hugging Face model repo
        if: ${{ env.HF_TOKEN != '' }}
        env:
          SYNC_MODE: ${{ steps.scope.outputs.sync_mode }}
        run: |
          if [[ "$SYNC_MODE" == "card" ]]; then
            python scripts/sync_huggingface.py --repo . --repo-id Stevesolun/ctx --repo-type model --card-only
          else
            python scripts/sync_huggingface.py --repo . --repo-id Stevesolun/ctx --repo-type model
          fi