download_models.py

import os
import urllib.request
from huggingface_hub import snapshot_download

def download_file(url, filename):
    print(f"Downloading {filename} from {url}...")
    try:
        urllib.request.urlretrieve(url, filename)
        print(f"Successfully downloaded {filename}")
    except Exception as e:
        print(f"Error downloading {filename}: {e}")
        raise

def download_hf_models():
    # Only download files, do not load into memory
    
    # PEGUSUS
    pegasus_name = "google/pegasus-cnn_dailymail"
    print(f"Downloading {pegasus_name} (snapshot)...")
    snapshot_download(repo_id=pegasus_name)
    
    # NLLB
    nllb_name = "facebook/nllb-200-distilled-1.3B"
    print(f"Downloading {nllb_name} (snapshot)...")
    snapshot_download(repo_id=nllb_name)
    
    print("All Hugging Face models downloaded and cached.")

if __name__ == "__main__":
    # Download lid.176.bin
    fasttext_url = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin"
    download_file(fasttext_url, "lid.176.bin")
    
    # Download HF Models
    download_hf_models()