.github/workflows/crawl.yml

name: Job Crawl

on:
  schedule:
    - cron: "0 1 * * *"   # 매일 오전 10시 KST (UTC+9 → UTC 01:00)
  workflow_dispatch:        # 수동 실행 허용

jobs:
  crawl:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: pip

      - name: Install dependencies
        run: pip install -r requirements.txt huggingface_hub

      # HF Dataset에서 이전 DB 복원 (누적 데이터 유지)
      - name: Restore DB from HF Dataset
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          HF_DATASET_REPO: ${{ secrets.HF_DATASET_REPO }}
        run: |
          python - <<'EOF'
          import os, shutil
          from huggingface_hub import hf_hub_download
          repo = os.environ["HF_DATASET_REPO"]
          token = os.environ["HF_TOKEN"]
          try:
              path = hf_hub_download(
                  repo_id=repo, filename="jobsonar.db",
                  repo_type="dataset", token=token,
              )
              os.makedirs("data", exist_ok=True)
              shutil.copy(path, "data/jobsonar.db")
              print("DB 복원 완료")
          except Exception as e:
              print(f"DB 없음 (첫 실행이면 정상): {e}")
          EOF

      - name: Run crawlers
        run: python -m crawler.run --source all --max-pages 15
        env:
          PYTHONPATH: ${{ github.workspace }}

      # 업데이트된 DB를 HF Dataset에 푸시 → Spaces가 자동으로 최신 데이터 사용
      - name: Push DB to HF Dataset
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          HF_DATASET_REPO: ${{ secrets.HF_DATASET_REPO }}
        run: |
          python - <<'EOF'
          import os
          from huggingface_hub import HfApi
          api = HfApi(token=os.environ["HF_TOKEN"])
          api.upload_file(
              path_or_fileobj="data/jobsonar.db",
              path_in_repo="jobsonar.db",
              repo_id=os.environ["HF_DATASET_REPO"],
              repo_type="dataset",
              commit_message="chore: weekly DB update",
          )
          print("HF Dataset 업로드 완료")
          EOF