.github/workflows/daily_scrape.yml

name: daily-scrape

on:
  schedule:
    - cron: "0 21 * * *"
  workflow_dispatch:

permissions:
  contents: write

jobs:
  scrape:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: "pip"               # L-08 FIX: cache pip dependencies -- saves 2-3 min/run

      - name: install dependencies
        run: pip install -r requirements.txt

      - name: setup neo4j schema
        # BUG-21 FIX: schema MUST run before scraping so the globalSearch
        # fulltext index exists when scrapers write interim data.
        # GraphLoader.setup_schema() uses IF NOT EXISTS so it is safe to run
        # on every workflow run.
        env:
          NEO4J_URI:      ${{ secrets.NEO4J_URI }}
          NEO4J_USER:     ${{ secrets.NEO4J_USER }}
          NEO4J_PASSWORD: ${{ secrets.NEO4J_PASSWORD }}
        run: |
          python3 -c "
          import os
          from neo4j import GraphDatabase
          uri  = os.environ.get('NEO4J_URI', '')
          user = os.environ.get('NEO4J_USER', 'neo4j')
          pwd  = os.environ.get('NEO4J_PASSWORD', '')
          if not uri:
              print('NEO4J_URI not set, skipping schema setup')
          else:
              from graph.loader import GraphLoader
              driver = GraphDatabase.driver(uri, auth=(user, pwd))
              loader = GraphLoader(driver=driver)
              loader.setup_schema()
              driver.close()
              print('Neo4j schema setup complete')
          "

      - name: run full pipeline
        env:
          DATAGOV_API_KEY:       ${{ secrets.DATAGOV_API_KEY }}
          NEO4J_URI:             ${{ secrets.NEO4J_URI }}
          NEO4J_USER:            ${{ secrets.NEO4J_USER }}
          NEO4J_PASSWORD:        ${{ secrets.NEO4J_PASSWORD }}
          OPENSANCTIONS_API_KEY: ${{ secrets.OPENSANCTIONS_API_KEY }}
        run: |
          python3 -m processing.pipeline \
            --scrapers cag,gem,pib,myneta,mca,loksabha,ed,wikidata,sebi,njdg,datagov,cvc,electoral_bond,icij,opensanctions,ibbi,ngo_darpan,cppp,ncrb,lgd

      - name: keep neo4j alive
        env:
          NEO4J_URI:      ${{ secrets.NEO4J_URI }}
          NEO4J_USER:     ${{ secrets.NEO4J_USER }}
          NEO4J_PASSWORD: ${{ secrets.NEO4J_PASSWORD }}
        run: |
          python3 -c "
          from neo4j import GraphDatabase
          import os
          uri  = os.environ.get('NEO4J_URI', '')
          user = os.environ.get('NEO4J_USER', 'neo4j')
          pwd  = os.environ.get('NEO4J_PASSWORD', '')
          if not uri:
              print('NEO4J_URI not set, skipping keep-alive')
          else:
              d = GraphDatabase.driver(uri, auth=(user, pwd))
              d.verify_connectivity()
              with d.session() as s:
                  r = s.run('MATCH (n) RETURN count(n) AS c').single()
                  print(f'Neo4j alive: {r[\"c\"]} nodes')
              d.close()
          "

      - name: load pipeline output into Neo4j
        env:
          NEO4J_URI:      ${{ secrets.NEO4J_URI }}
          NEO4J_USER:     ${{ secrets.NEO4J_USER }}
          NEO4J_PASSWORD: ${{ secrets.NEO4J_PASSWORD }}
          ADMIN_SECRET:   ${{ secrets.ADMIN_SECRET }}
        run: |
          API_BASE="${{ secrets.API_BASE_URL }}"
          API_BASE="${API_BASE:-https://abinazebinoly-bharatgraph.hf.space}"
          echo "Triggering graph load via API..."
          # NEW-A8 FIX: pass X-Admin-Secret header -- required after BUG-6 auth fix
          curl -s -X POST "${API_BASE}/admin/pipeline" \
            -H "Content-Type: application/json" \
            -H "X-Admin-Secret: ${ADMIN_SECRET}" \
            --max-time 30 || echo "Warning: pipeline trigger returned non-200 (may still be running)"
          echo "Graph load triggered. Check /admin/pipeline/status for progress."

      - name: commit updated data artifacts
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add data/processed/ || true
          git diff --staged --quiet || \
            git commit -m "chore(data): daily pipeline run $(date -u +%Y-%m-%d)"
          git push origin main || true
      # BUG-H3 FIX: moved to end so it covers ALL steps including load+commit
      - name: notify on failure
        if: failure()
        run: |
          echo "::error::Daily scrape pipeline FAILED at $(date -u +%Y-%m-%dT%H:%MZ)"
          echo "Check /admin/pipeline/status and GitHub Actions logs for details"