verify_langsmith.py

#!/usr/bin/env python3
"""
LangSmith Configuration Verification Script
Tests all aspects of LangSmith setup and connectivity
"""

import os
import sys
from datetime import datetime
from dotenv import load_dotenv

def load_environment():
    """Load environment variables from .env file"""
    print("🔧 Loading Environment Configuration...")
    print("-" * 50)
    
    # Load .env file
    env_file = '.env'
    if os.path.exists(env_file):
        load_dotenv(env_file)
        print(f"✅ Loaded {env_file}")
    else:
        print(f"❌ {env_file} file not found")
        return False
    
    return True

def check_environment_variables():
    """Check all LangSmith-related environment variables"""
    print("\n🔍 Checking Environment Variables...")
    print("-" * 50)
    
    required_vars = {
        'LANGCHAIN_API_KEY': 'LangSmith API Key',
        'LANGCHAIN_TRACING_V2': 'Tracing Enable Flag',
        'LANGCHAIN_PROJECT': 'Project Name',
        'LANGCHAIN_ENDPOINT': 'API Endpoint'
    }
    
    env_status = {}
    
    for var, description in required_vars.items():
        value = os.getenv(var)
        if value:
            if var == 'LANGCHAIN_API_KEY':
                # Mask API key for security
                display_value = f"{value[:8]}...{value[-8:]}" if len(value) > 16 else "***MASKED***"
            else:
                display_value = value
            print(f"✅ {var}: {display_value}")
            env_status[var] = True
        else:
            print(f"❌ {var}: NOT SET ({description})")
            env_status[var] = False
    
    return env_status

def setup_langsmith_from_agent():
    """Import and run the setup function from agent.py"""
    print("\n🔧 Running LangSmith Setup from agent.py...")
    print("-" * 50)
    
    try:
        # Import the setup function
        from agent import setup_langsmith, langsmith_enabled
        
        print(f"✅ Agent module imported successfully")
        print(f"🔄 LangSmith setup result: {'✅ ENABLED' if langsmith_enabled else '❌ DISABLED'}")
        
        # Check environment variables after setup
        print("\n📋 Environment variables after agent setup:")
        env_vars_after = {
            'LANGCHAIN_TRACING_V2': os.getenv('LANGCHAIN_TRACING_V2'),
            'LANGCHAIN_ENDPOINT': os.getenv('LANGCHAIN_ENDPOINT'),
            'LANGCHAIN_PROJECT': os.getenv('LANGCHAIN_PROJECT'),
            'LANGCHAIN_API_KEY': '***SET***' if os.getenv('LANGCHAIN_API_KEY') else 'NOT_SET'
        }
        
        for var, value in env_vars_after.items():
            status = "✅" if value and value != 'NOT_SET' else "❌"
            print(f"{status} {var}: {value}")
            
        return langsmith_enabled
        
    except Exception as e:
        print(f"❌ Error importing agent module: {e}")
        return False

def test_langsmith_import():
    """Test LangSmith module import"""
    print("\n📦 Testing LangSmith Module Import...")
    print("-" * 50)
    
    try:
        import langsmith
        print(f"✅ langsmith module imported successfully")
        print(f"📋 Version: {getattr(langsmith, '__version__', 'Unknown')}")
        
        # Test traceable import
        from langsmith import traceable
        print(f"✅ @traceable decorator imported successfully")
        
        return True
        
    except ImportError as e:
        print(f"❌ Failed to import langsmith: {e}")
        print("💡 Try: pip install langsmith")
        return False
    except Exception as e:
        print(f"❌ Error with langsmith module: {e}")
        return False

def test_langsmith_connection():
    """Test connection to LangSmith API"""
    print("\n🌐 Testing LangSmith API Connection...")
    print("-" * 50)
    
    try:
        from langsmith import Client
        
        # Get API key
        api_key = os.getenv('LANGCHAIN_API_KEY')
        if not api_key or api_key == 'your_langsmith_api_key_here':
            print("❌ No valid API key found")
            return False
        
        # Create client
        client = Client()
        
        # Test basic connection by getting project info
        print("🔄 Testing API connection...")
        
        # This will test the connection
        try:
            projects = list(client.list_projects(limit=1))
            print("✅ Successfully connected to LangSmith API")
            print(f"📊 API connection test passed")
            return True
            
        except Exception as api_error:
            print(f"❌ API Connection failed: {api_error}")
            if "401" in str(api_error) or "authentication" in str(api_error).lower():
                print("💡 Check your API key - it might be invalid")
            elif "403" in str(api_error) or "forbidden" in str(api_error).lower():
                print("💡 API key valid but lacks permissions")
            else:
                print("💡 Network or server issue")
            return False
            
    except ImportError:
        print("❌ LangSmith client not available")
        return False
    except Exception as e:
        print(f"❌ Connection test error: {e}")
        return False

def test_traceable_decorator():
    """Test @traceable decorator functionality"""
    print("\n🎯 Testing @traceable Decorator...")
    print("-" * 50)
    
    try:
        from langsmith import traceable
        
        # Define a test function with @traceable
        @traceable(name="test_function")
        def test_traced_function(x, y):
            """Test function for tracing"""
            result = x + y
            return f"Sum of {x} and {y} is {result}"
        
        # Test the function
        print("🔄 Testing traced function...")
        result = test_traced_function(3, 5)
        print(f"✅ Traced function executed: {result}")
        
        # Note: Actual tracing verification requires checking LangSmith dashboard
        print("📝 Note: Check your LangSmith dashboard to verify traces appear")
        print("🔗 Dashboard: https://smith.langchain.com")
        
        return True
        
    except Exception as e:
        print(f"❌ Traceable test error: {e}")
        return False

def check_project_configuration():
    """Check project-specific configuration"""
    print("\n🏗️ Checking Project Configuration...")
    print("-" * 50)
    
    project_name = os.getenv('LANGCHAIN_PROJECT', 'default')
    endpoint = os.getenv('LANGCHAIN_ENDPOINT', 'https://api.smith.langchain.com')
    tracing = os.getenv('LANGCHAIN_TRACING_V2', 'false')
    
    print(f"📋 Project Name: {project_name}")
    print(f"🔗 API Endpoint: {endpoint}")
    print(f"🔍 Tracing Enabled: {tracing}")
    
    # Validate configuration
    config_ok = True
    
    if project_name == 'default':
        print("⚠️  Using default project name")
    else:
        print(f"✅ Custom project configured: {project_name}")
    
    if endpoint != 'https://api.smith.langchain.com':
        print(f"⚠️  Non-standard endpoint: {endpoint}")
    else:
        print("✅ Standard LangSmith endpoint")
    
    if tracing.lower() != 'true':
        print("❌ Tracing is disabled")
        config_ok = False
    else:
        print("✅ Tracing is enabled")
    
    return config_ok

def main():
    """Run all LangSmith verification tests"""
    print("🚀 LangSmith Configuration Verification")
    print(f"⏰ Started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print("=" * 60)
    
    # Test results
    results = {}
    
    # Load environment
    results['env_loaded'] = load_environment()
    
    # Check environment variables
    env_status = check_environment_variables()
    results['env_vars'] = all(env_status.values())
    
    # Test module import
    results['module_import'] = test_langsmith_import()
    
    # Setup from agent.py
    results['agent_setup'] = setup_langsmith_from_agent()
    
    # Test API connection
    results['api_connection'] = test_langsmith_connection()
    
    # Test traceable decorator
    results['traceable_test'] = test_traceable_decorator()
    
    # Check project configuration
    results['project_config'] = check_project_configuration()
    
    # Summary
    print("\n" + "=" * 60)
    print("📋 VERIFICATION SUMMARY")
    print("=" * 60)
    
    for test, passed in results.items():
        status = "✅ PASS" if passed else "❌ FAIL"
        test_name = test.replace('_', ' ').title()
        print(f"{status} {test_name}")
    
    overall_success = all(results.values())
    
    if overall_success:
        print(f"\n🎉 LangSmith is properly configured and working!")
        print(f"🔗 View traces at: https://smith.langchain.com")
        print(f"📊 Project: {os.getenv('LANGCHAIN_PROJECT', 'gaia-agent-project')}")
    else:
        print(f"\n⚠️  LangSmith configuration has issues")
        print(f"💡 Review the failed tests above for solutions")
        
        # Specific recommendations
        if not results.get('env_vars'):
            print("🔧 Fix: Set missing environment variables in .env file")
        if not results.get('api_connection'):
            print("🔧 Fix: Check API key validity at https://smith.langchain.com")
        if not results.get('agent_setup'):
            print("🔧 Fix: Check agent.py setup_langsmith() function")
    
    print(f"\n⏰ Completed at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    
    return overall_success

if __name__ == "__main__":
    success = main()
    sys.exit(0 if success else 1)