Spaces:
Running
Running
Download scripts/update_circulars.py from gauravmeena0708/epfo-circulars: direct link, hf CLI and curl.
- Browser
- Download file 5.22 kB
-
https://huggingface.co/spaces/gauravmeena0708/epfo-circulars/resolve/main/scripts/update_circulars.py
- Command line
-
hf download hf://spaces/gauravmeena0708/epfo-circulars/scripts/update_circulars.py
-
curl -L -o update_circulars.py https://huggingface.co/spaces/gauravmeena0708/epfo-circulars/resolve/main/scripts/update_circulars.py
5.22 kB
| """Bring the circular list and the search index up to date, from a machine that can reach the issuer's site. | |
| python scripts/update_circulars.py # list, index, commit, push (a push to main deploys the Space) | |
| python scripts/update_circulars.py --no-push # stop after the local commits | |
| python scripts/update_circulars.py --check # report what is new or waiting; change nothing | |
| GitHub's hosted runners receive 403 from the issuer's site, so this runs locally. Steps: | |
| fetch.py adds newly listed circulars to circular-data.json; index_new_circulars.py reads their | |
| PDFs and adds them to the index; the index commit is pushed; deploy/runtime-assets.json is then | |
| pinned to that commit and pushed, which is the order the Space build needs. | |
| """ | |
| import argparse | |
| from pathlib import Path | |
| import subprocess | |
| import sys | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT)) | |
| sys.path.insert(0, str(ROOT / 'scripts')) | |
| INDEX_PATHS = ['ocr_updates', *(f'vector_store/data_index/{name}' for name in | |
| ('faiss_index.index', 'faiss_index.passages.db', 'faiss_index.bm25.json.gz'))] | |
| def git(*args, capture=False): | |
| result = subprocess.run(['git', *args], cwd=ROOT, check=True, text=True, | |
| stdout=subprocess.PIPE if capture else None) | |
| return result.stdout.strip() if capture else '' | |
| def commit(paths, message): | |
| """Commit the given paths if they changed; return True when a commit was made.""" | |
| git('add', '--', *paths) | |
| if subprocess.run(['git', 'diff', '--staged', '--quiet'], cwd=ROOT).returncode == 0: | |
| return False | |
| git('commit', '-q', '-m', message) | |
| return True | |
| def run(push=True, min_pending=1, limit=0): | |
| import fetch | |
| import fetch_runtime_assets | |
| import index_new_circulars as indexer | |
| if git('status', '--porcelain', capture=True): | |
| raise SystemExit('The working tree has uncommitted changes; commit or stash them first.') | |
| branch = git('rev-parse', '--abbrev-ref', 'HEAD', capture=True) | |
| if branch != 'main': | |
| raise SystemExit(f'Run this on main (currently on {branch}).') | |
| git('pull', '--ff-only', '-q') | |
| added = fetch.fetch_circular_metadata(data_file=str(ROOT / fetch.CIRCULAR_DATA_FILE)) | |
| listed = commit([fetch.CIRCULAR_DATA_FILE], 'Update EPF circulars data') | |
| waiting = indexer.pending(ROOT) | |
| print(f'{len(added)} newly listed; {len(waiting)} waiting to be indexed') | |
| indexed = False | |
| if waiting and len(waiting) >= min_pending: | |
| texts, failed = indexer.read_circulars(waiting[:limit] if limit else waiting) | |
| if texts: | |
| indexer.write_updates(ROOT / 'ocr_updates', texts) | |
| before, after = indexer.index_updates(ROOT) | |
| print(f'Index checked: {before} -> {after} passages; earlier passages unchanged.') | |
| indexed = commit(INDEX_PATHS, f'Index {len(texts)} newly listed circulars') | |
| if failed: | |
| print(f'{len(failed)} PDFs could not be read and stay waiting for the next run.') | |
| elif waiting: | |
| print(f'Indexing starts at {min_pending} waiting circulars.') | |
| if not (listed or indexed): | |
| print('Nothing to commit.') | |
| return | |
| if not push: | |
| print('Committed locally, not pushed. After pushing an index commit, run ' | |
| '"python scripts/fetch_runtime_assets.py --record-commit <its full sha>" and commit the manifest.') | |
| return | |
| git('push', '-q') | |
| if indexed: | |
| # The Space build downloads the index at the pinned commit, so the commit must be on GitHub first. | |
| fetch_runtime_assets.record(git('rev-parse', 'HEAD', capture=True), root=ROOT, | |
| path=ROOT / fetch_runtime_assets.MANIFEST) | |
| commit([str(fetch_runtime_assets.MANIFEST).replace('\\', '/')], 'Pin runtime assets to the updated index') | |
| git('push', '-q') | |
| print('Pushed. The sync workflow now deploys the Space.') | |
| def check(): | |
| import fetch | |
| import index_new_circulars as indexer | |
| added = fetch.fetch_circular_metadata(data_file=str(ROOT / fetch.CIRCULAR_DATA_FILE), dry_run=True) | |
| waiting = indexer.pending(ROOT) | |
| print(f'{len(added)} newly listed (not written); {len(waiting)} already listed and waiting to be indexed') | |
| def main(): | |
| if hasattr(sys.stdout, 'reconfigure'): | |
| sys.stdout.reconfigure(encoding='utf-8', errors='replace') | |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument('--check', action='store_true', help='Report only; change nothing') | |
| parser.add_argument('--no-push', action='store_true', help='Commit locally without pushing') | |
| parser.add_argument('--min-pending', type=int, default=1, | |
| help='Index only when at least this many circulars are waiting (each index update adds about 290 MB to Git LFS)') | |
| parser.add_argument('--limit', type=int, default=0, help='Read at most this many circulars (0 = all)') | |
| args = parser.parse_args() | |
| if args.check: | |
| return check() | |
| run(push=not args.no_push, min_pending=args.min_pending, limit=args.limit) | |
| if __name__ == '__main__': | |
| sys.exit(main()) | |