import sys import os import time from huggingface_hub import InferenceClient # Add the project root to sys.path sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) from backend.app.config import config def test_hf_inference(): print("Testing Hugging Face Inference...") print(f"Model: {config.LLM_MODEL}") print(f"API Key Present: {'Yes' if config.HUGGINGFACE_API_KEY else 'No'}") # Test 1: Intended Model (Llama 3) print(f"\n--- Testing {config.LLM_MODEL} ---") # Use standard Inference API URL for better compatibility checks # model_url = f"https://api-inference.huggingface.co/models/{config.LLM_MODEL}" # Sticking to what the app uses to be sure model_url = f"https://router.huggingface.co/hf-inference/models/{config.LLM_MODEL}" client = InferenceClient(model=model_url, token=config.HUGGINGFACE_API_KEY) prompt = "Hello!" try: response = client.text_generation(prompt, max_new_tokens=10) print("Success!") except Exception as e: print(f"FAILED with {config.LLM_MODEL}") if hasattr(e, 'response'): print(f"Status: {e.response.status_code}") print(f"Body: {e.response.text}") else: print(f"Error: {e}") # Test 2: Fallback Non-Gated Model (Zephyr 7B Beta) - Standard Endpoint fallback_model = "HuggingFaceH4/zephyr-7b-beta" print(f"\n--- Testing Fallback {fallback_model} (Standard API) ---") model_url_fallback = f"https://api-inference.huggingface.co/models/{fallback_model}" # Try with Token first client_fallback = InferenceClient(model=model_url_fallback, token=config.HUGGINGFACE_API_KEY) try: response = client_fallback.text_generation(prompt, max_new_tokens=10) print(f"Success with Token! Generated: {response}") except Exception as e: print(f"FAILED with Token: {e}") # Test 3: Library Managed URL (Zephyr) print(f"\n--- Testing Zephyr (Library Managed URL) ---") try: # Let the library decide the URL client_lib = InferenceClient(model="HuggingFaceH4/zephyr-7b-beta", token=config.HUGGINGFACE_API_KEY) response = client_lib.text_generation(prompt, max_new_tokens=10) print(f"Success (with token)! Generated: {response}") except Exception as e: print(f"FAILED (with token): {e}") # Test 5: GPT-2 (Anonymous) print(f"\n--- Testing GPT-2 (Anonymous) ---") try: client_gpt2 = InferenceClient(model="gpt2", token=None) response = client_gpt2.text_generation("Hello world", max_new_tokens=10) print(f"Success GPT-2! Generated: {response}") except Exception as e: print(f"FAILED GPT-2: {e}") if __name__ == "__main__": test_hf_inference()