scripts/download_indicbert_simple.py

#!/usr/bin/env python
"""
Simple script to download IndicBERT using huggingface_hub (no transformers needed).

This is the LIGHTEST method - only needs huggingface_hub library.

Usage:
    # Install: pip install huggingface_hub
    # Then run:
    python scripts/download_indicbert_simple.py your_token_here
"""

import sys
import os
import time
from pathlib import Path


def download_with_hub(token: str):
    """Download IndicBERT using huggingface_hub (lightweight method)."""
    print("=" * 60)
    print("Downloading IndicBERT with huggingface_hub")
    print("=" * 60)
    print(f"Token: {token[:10]}...{token[-4:]}")
    print()
    
    try:
        from huggingface_hub import snapshot_download, login
        
        model_id = "ai4bharat/indic-bert"
        
        # Step 1: Login
        print("Step 1: Authenticating...")
        try:
            login(token=token, add_to_git_credential=True)
            print("  ✓ Authentication successful")
        except Exception as e:
            print(f"  ⚠ Login warning: {e}")
            print("  Continuing with token in request...")
        
        # Step 2: Download
        print(f"\nStep 2: Downloading model '{model_id}'...")
        print("  This may take several minutes (model is ~500MB)...")
        
        start = time.time()
        
        # Download to default cache location
        local_path = snapshot_download(
            repo_id=model_id,
            token=token,
            local_dir_use_symlinks=False,
            resume_download=True  # Resume if interrupted
        )
        
        download_time = time.time() - start
        
        print(f"\n  ✓ Download completed!")
        print(f"  ✓ Time: {download_time:.2f}s ({download_time/60:.1f} minutes)")
        print(f"  ✓ Location: {local_path}")
        
        # Verify files
        model_path = Path(local_path)
        files = list(model_path.rglob("*"))
        files = [f for f in files if f.is_file()]
        
        total_size = sum(f.stat().st_size for f in files)
        size_mb = total_size / (1024 * 1024)
        
        print(f"  ✓ Files: {len(files)}")
        print(f"  ✓ Size: {size_mb:.1f} MB")
        
        # Check for key files
        key_files = ["config.json", "pytorch_model.bin", "tokenizer.json"]
        found_files = []
        for key_file in key_files:
            if (model_path / key_file).exists():
                found_files.append(key_file)
        
        if found_files:
            print(f"  ✓ Key files found: {', '.join(found_files)}")
        else:
            # Check for safetensors
            safetensors = list(model_path.glob("*.safetensors"))
            if safetensors:
                print(f"  ✓ Found safetensors files: {len(safetensors)}")
        
        print("\n" + "=" * 60)
        print("SUCCESS: IndicBERT downloaded successfully!")
        print("=" * 60)
        print(f"\nModel location: {local_path}")
        print("\nTo use the model later with transformers:")
        print(f"  from transformers import AutoModel, AutoTokenizer")
        print(f"  model = AutoModel.from_pretrained('{local_path}')")
        print(f"  tokenizer = AutoTokenizer.from_pretrained('{local_path}')")
        print("\nOr use the model ID (will use cached files):")
        print(f"  model = AutoModel.from_pretrained('{model_id}', token='{token[:10]}...')")
        
        return True
        
    except ImportError:
        print("\n[ERROR] huggingface_hub not installed!")
        print("\nInstall it with:")
        print("  pip install huggingface_hub")
        print("\nThis is a lightweight library (no transformers needed)")
        return False
        
    except Exception as e:
        error_msg = str(e)
        print(f"\n[ERROR] Download failed: {error_msg}")
        
        # Provide specific error guidance
        if "401" in error_msg or "unauthorized" in error_msg.lower():
            print("\n🔐 Authentication Error:")
            print("  1. Check your token is correct")
            print("  2. Token should start with 'hf_'")
            print("  3. Get token from: https://huggingface.co/settings/tokens")
            
        elif "gated" in error_msg.lower() or "access" in error_msg.lower():
            print("\n🔒 Access Required:")
            print("  1. Request access at: https://huggingface.co/ai4bharat/indic-bert")
            print("  2. Click 'Agree and access repository'")
            print("  3. Wait for approval (usually instant)")
            print("  4. Then run this script again")
            
        elif "404" in error_msg or "not found" in error_msg.lower():
            print("\n❌ Model Not Found:")
            print("  Check the model ID: ai4bharat/indic-bert")
            print("  Verify it exists at: https://huggingface.co/ai4bharat/indic-bert")
            
        else:
            print("\n💡 Troubleshooting:")
            print("  1. Check internet connection")
            print("  2. Try again (may be temporary network issue)")
            print("  3. Check token permissions")
        
        return False


def main():
    """Main entry point."""
    # Get token
    token = None
    
    # From command line
    if len(sys.argv) > 1:
        token = sys.argv[1]
    # From environment
    elif os.getenv("HUGGINGFACE_TOKEN"):
        token = os.getenv("HUGGINGFACE_TOKEN")
    
    if not token:
        print("=" * 60)
        print("IndicBERT Download Script")
        print("=" * 60)
        print("\n❌ ERROR: No token provided!")
        print("\nUsage:")
        print("  python scripts/download_indicbert_simple.py your_token_here")
        print("\nOr set environment variable:")
        print("  set HUGGINGFACE_TOKEN=your_token_here")
        print("  python scripts/download_indicbert_simple.py")
        print("\nGet your token from:")
        print("  https://huggingface.co/settings/tokens")
        return 1
    
    # Validate token format
    if not token.startswith("hf_"):
        print("⚠ WARNING: Token should start with 'hf_'")
        print("  Make sure you're using a HuggingFace access token")
        response = input("Continue anyway? (y/n): ")
        if response.lower() != 'y':
            return 1
    
    success = download_with_hub(token)
    return 0 if success else 1


if __name__ == "__main__":
    exit_code = main()
    sys.exit(exit_code)