scripts/check_hf_token.py

import sys
from pathlib import Path

# Add project root to Python path
project_root = Path(__file__).parent.parent
sys.path.append(str(project_root))

from config.settings import Settings
from huggingface_hub import HfApi

def check_dataset_files():
    """Check files available in the HuggingFace dataset"""
    print(f"\nChecking dataset: {Settings.HF_DATASET}")
    print(f"Using token: {'Present' if Settings.HF_TOKEN else 'Missing'}")
    
    api = HfApi(token=Settings.HF_TOKEN)
    try:
        files = api.list_repo_files(Settings.HF_DATASET, repo_type="dataset")
        print("\nFiles in dataset:")
        for f in files:
            print(f"- {f}")
    except Exception as e:
        print(f"\nError accessing dataset: {type(e).__name__}")
        print(f"Error details: {str(e)}")

if __name__ == "__main__":
    check_dataset_files()