#!/usr/bin/env python """ Quick test script to download IndicBERT with a token. Usage: # Option 1: Set environment variable set HUGGINGFACE_TOKEN=your_token_here python scripts/test_indicbert_download.py # Option 2: Pass token as argument python scripts/test_indicbert_download.py your_token_here """ import sys import os import time def test_indicbert_download(token: str = None): """Test IndicBERT download with token.""" if token is None: token = os.getenv("HUGGINGFACE_TOKEN") if not token: print("ERROR: No token provided!") print("\nPlease provide token in one of these ways:") print("1. Set environment variable: set HUGGINGFACE_TOKEN=your_token") print("2. Pass as argument: python scripts/test_indicbert_download.py your_token") return False print("=" * 60) print("Testing IndicBERT Download with Token") print("=" * 60) print(f"Token: {token[:10]}...{token[-4:]}") print() try: from transformers import AutoModel, AutoTokenizer model_name = "ai4bharat/indic-bert" print("Step 1: Loading tokenizer...") start = time.time() tokenizer = AutoTokenizer.from_pretrained(model_name, token=token) print(f" ✓ Tokenizer loaded in {time.time() - start:.2f}s") print("\nStep 2: Loading model (this may take a few minutes)...") start = time.time() model = AutoModel.from_pretrained(model_name, token=token) load_time = time.time() - start print(f" ✓ Model loaded in {load_time:.2f}s") print("\nStep 3: Testing model functionality...") test_text = "Test message for scam detection" inputs = tokenizer(test_text, return_tensors="pt", truncation=True, max_length=512) model.eval() outputs = model(**inputs) print(f" ✓ Model processed text successfully") print(f" ✓ Output shape: {outputs.last_hidden_state.shape}") print("\n" + "=" * 60) print("SUCCESS: IndicBERT downloaded and tested!") print(f"Load time: {load_time:.2f}s") if load_time < 10.0: print("✓ Meets requirement: <10 seconds") else: print("⚠ Exceeds requirement: >10 seconds (may be first download)") print("=" * 60) return True except ImportError as e: print(f"ERROR: transformers not installed: {e}") print("Run: pip install transformers") return False except Exception as e: error_msg = str(e) print(f"ERROR: {error_msg}") if "gated repo" in error_msg.lower() or "access" in error_msg.lower(): print("\nThis model requires:") print("1. Request access at: https://huggingface.co/ai4bharat/indic-bert") print("2. Make sure your token has access to this repository") print("3. Token should start with 'hf_'") elif "401" in error_msg or "unauthorized" in error_msg.lower(): print("\nAuthentication failed. Please check:") print("1. Token is correct") print("2. Token has access to ai4bharat/indic-bert") print("3. You've requested access to the model") return False if __name__ == "__main__": # Get token from command line argument or environment token = None if len(sys.argv) > 1: token = sys.argv[1] success = test_indicbert_download(token) sys.exit(0 if success else 1)