Spaces:

sdilbaz
/

prime_py

Sleeping

prime_py / debug_data_loading.py

Add debug script for data loading diagnostics

1b87393 10 months ago

3.06 kB

"""Debug script to diagnose data loading issues on Hugging Face Spaces."""
from pathlib import Path
import sys

print("=" * 80)
print("DATA LOADING DEBUG")
print("=" * 80)

# Check current working directory
cwd = Path.cwd()
print(f"\nCurrent working directory: {cwd}")
print(f"CWD exists: {cwd.exists()}")

# Check data directory
data_dir = cwd / "data"
print(f"\nData directory: {data_dir}")
print(f"Data dir exists: {data_dir.exists()}")

if data_dir.exists():
    print(f"\nContents of {data_dir}:")
    try:
        for item in sorted(data_dir.iterdir()):
            size = item.stat().st_size if item.is_file() else "DIR"
            print(f"  {item.name:50} {size}")
    except Exception as e:
        print(f"  Error listing: {e}")

# Check for globocan files
print(f"\nSearching for globocan*.xlsx files:")
globocan_files = list(data_dir.glob("globocan*.xlsx")) if data_dir.exists() else []
if globocan_files:
    for gf in globocan_files:
        print(f"  Found: {gf}")
        print(f"    Size: {gf.stat().st_size} bytes")
        print(f"    Exists: {gf.exists()}")
        print(f"    Is file: {gf.is_file()}")
        
        # Check if it's an LFS pointer
        if gf.stat().st_size < 200:  # LFS pointers are typically ~130 bytes
            print(f"    WARNING: File is suspiciously small - might be LFS pointer!")
            try:
                with open(gf, 'rb') as f:
                    first_bytes = f.read(100)
                    if b'version https://git-lfs.github.com' in first_bytes:
                        print(f"    ERROR: This IS an LFS pointer file, not the actual data!")
                    print(f"    First bytes: {first_bytes[:50]}")
            except Exception as e:
                print(f"    Error reading: {e}")
        
        # Try to check if it's a valid Excel file
        try:
            import pandas as pd
            xl = pd.ExcelFile(gf)
            print(f"    Sheets: {xl.sheet_names}")
            print(f"    ✓ Valid Excel file")
        except Exception as e:
            print(f"    ✗ Error reading as Excel: {e}")
else:
    print("  No globocan files found!")

# Check DataRegistry
print(f"\nTesting DataRegistry:")
try:
    sys.path.insert(0, str(cwd / "src"))
    from primepy.data_registry import DataRegistry
    
    reg = DataRegistry(cwd)
    reg.default_from_repo()
    print(f"  Registered datasets: {list(reg._registry.keys())}")
    
    for year, paths in reg._registry.items():
        print(f"  Year {year}:")
        print(f"    xlsx_bundle: {paths.xlsx_bundle}")
        if paths.xlsx_bundle:
            print(f"    exists: {paths.xlsx_bundle.exists()}")
            print(f"    size: {paths.xlsx_bundle.stat().st_size if paths.xlsx_bundle.exists() else 'N/A'}")
    
    # Try to load
    print(f"\n  Attempting to load 2020 dataset...")
    tables = reg.load("2020")
    print(f"  ✓ Successfully loaded! Tables: {list(tables.keys())}")
    
except Exception as e:
    print(f"  ✗ Error: {e}")
    import traceback
    traceback.print_exc()

print("\n" + "=" * 80)

	"""Debug script to diagnose data loading issues on Hugging Face Spaces."""
	from pathlib import Path
	import sys

	print("=" * 80)
	print("DATA LOADING DEBUG")
	print("=" * 80)

	# Check current working directory
	cwd = Path.cwd()
	print(f"\nCurrent working directory: {cwd}")
	print(f"CWD exists: {cwd.exists()}")

	# Check data directory
	data_dir = cwd / "data"
	print(f"\nData directory: {data_dir}")
	print(f"Data dir exists: {data_dir.exists()}")

	if data_dir.exists():
	print(f"\nContents of {data_dir}:")
	try:
	for item in sorted(data_dir.iterdir()):
	size = item.stat().st_size if item.is_file() else "DIR"
	print(f" {item.name:50} {size}")
	except Exception as e:
	print(f" Error listing: {e}")

	# Check for globocan files
	print(f"\nSearching for globocan*.xlsx files:")
	globocan_files = list(data_dir.glob("globocan*.xlsx")) if data_dir.exists() else []
	if globocan_files:
	for gf in globocan_files:
	print(f" Found: {gf}")
	print(f" Size: {gf.stat().st_size} bytes")
	print(f" Exists: {gf.exists()}")
	print(f" Is file: {gf.is_file()}")

	# Check if it's an LFS pointer
	if gf.stat().st_size < 200: # LFS pointers are typically ~130 bytes
	print(f" WARNING: File is suspiciously small - might be LFS pointer!")
	try:
	with open(gf, 'rb') as f:
	first_bytes = f.read(100)
	if b'version https://git-lfs.github.com' in first_bytes:
	print(f" ERROR: This IS an LFS pointer file, not the actual data!")
	print(f" First bytes: {first_bytes[:50]}")
	except Exception as e:
	print(f" Error reading: {e}")

	# Try to check if it's a valid Excel file
	try:
	import pandas as pd
	xl = pd.ExcelFile(gf)
	print(f" Sheets: {xl.sheet_names}")
	print(f" ✓ Valid Excel file")
	except Exception as e:
	print(f" ✗ Error reading as Excel: {e}")
	else:
	print(" No globocan files found!")

	# Check DataRegistry
	print(f"\nTesting DataRegistry:")
	try:
	sys.path.insert(0, str(cwd / "src"))
	from primepy.data_registry import DataRegistry

	reg = DataRegistry(cwd)
	reg.default_from_repo()
	print(f" Registered datasets: {list(reg._registry.keys())}")

	for year, paths in reg._registry.items():
	print(f" Year {year}:")
	print(f" xlsx_bundle: {paths.xlsx_bundle}")
	if paths.xlsx_bundle:
	print(f" exists: {paths.xlsx_bundle.exists()}")
	print(f" size: {paths.xlsx_bundle.stat().st_size if paths.xlsx_bundle.exists() else 'N/A'}")

	# Try to load
	print(f"\n Attempting to load 2020 dataset...")
	tables = reg.load("2020")
	print(f" ✓ Successfully loaded! Tables: {list(tables.keys())}")

	except Exception as e:
	print(f" ✗ Error: {e}")
	import traceback
	traceback.print_exc()

	print("\n" + "=" * 80)