Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
File size: 2,155 Bytes
b5a9373 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
import subprocess
import sys
from huggingface_upload import upload_all_to_huggingface
all_scripts = [
"data_updating_scripts/get_data.py",
"data_updating_scripts/fix_pdf_bills.py",
"data_updating_scripts/known_bills_status.py",
"data_updating_scripts/migrate_iapp_categories.py",
"data_updating_scripts/mark_no_text_bills.py",
"data_updating_scripts/generate_summaries.py",
"data_updating_scripts/generate_suggested_questions.py",
"data_updating_scripts/generate_reports.py",
"data_updating_scripts/eu_vectorstore.py",
]
print("Do you want to pull new data from LegiScan?")
print("Enter 'y' or 'yes' to pull new data, or 'n' or 'no' to skip and use existing data:")
response = input().strip().lower()
if response in ["y", "yes"]:
print("\n✓ Will pull new data from LegiScan")
scripts_to_run = all_scripts
elif response in ["n", "no"]:
print("\n✓ Skipping data pull, using existing data")
scripts_to_run = all_scripts[2:]
else:
print(f"\n✗ Invalid response '{response}'. Please run the script again and enter 'y' or 'n'.")
sys.exit(1)
print(f"\nWill run {len(scripts_to_run)} scripts:")
for script in scripts_to_run:
print(f" - {script}")
print("\n" + "=" * 50)
for script in scripts_to_run:
print(f"\n--- Running {script} ---")
print("=" * 50)
result = subprocess.run([sys.executable, script])
if result.returncode != 0:
print(f"\n✗ Script {script} failed with return code {result.returncode}")
print("Do you want to continue with the remaining scripts? (y/n):")
continue_response = input().strip().lower()
if continue_response not in ["y", "yes"]:
print("Stopping pipeline execution.")
sys.exit(1)
else:
print(f"✓ {script} completed successfully")
print("\n" + "=" * 50)
print("✓ Pipeline execution completed!")
print("\nUploading all JSON datasets to HuggingFace…")
try:
upload_all_to_huggingface()
print("✓ HuggingFace upload complete!")
except KeyError as e:
print(f"✗ HuggingFace config error: {e}")
except Exception as e:
print(f"✗ Upload failed: {e}")
|