backend/runner/config_old.py

"""
Unified configuration for Hugging Face datasets integration.
All runner modules should import from this module instead of defining their own paths.
"""

import os
import json
from pathlib import Path
from typing import Any, Dict, Optional, List, Tuple

# Try to import required libraries
try:
    from datasets import load_dataset
    DATASETS_AVAILABLE = True
except ImportError:
    print("⚠️  datasets library not available - HF dataset loading disabled")
    DATASETS_AVAILABLE = False

try:
    from huggingface_hub import hf_hub_download
    HF_HUB_AVAILABLE = True
except ImportError:
    print("⚠️  huggingface_hub library not available - HF file loading disabled")
    HF_HUB_AVAILABLE = False

# Environment variables for dataset names
ARTEFACT_JSON_DATASET = os.getenv('ARTEFACT_JSON_DATASET', 'samwaugh/artefact-json')
ARTEFACT_EMBEDDINGS_DATASET = os.getenv('ARTEFACT_EMBEDDINGS_DATASET', 'samwaugh/artefact-embeddings')
ARTEFACT_MARKDOWN_DATASET = os.getenv('ARTEFACT_MARKDOWN_DATASET', 'samwaugh/artefact-markdown')

# Legacy path variables for backward compatibility
JSON_INFO_DIR = "/data/hub/datasets--samwaugh--artefact-json/snapshots/latest"
EMBEDDINGS_DIR = "/data/hub/datasets--samwaugh--artefact-embeddings/snapshots/latest"
MARKDOWN_DIR = "/data/hub/datasets--samwaugh--artefact-markdown/snapshots/latest"

# Embedding file paths for backward compatibility
CLIP_EMBEDDINGS_ST = Path(EMBEDDINGS_DIR) / "clip_embeddings.safetensors"
PAINTINGCLIP_EMBEDDINGS_ST = Path(EMBEDDINGS_DIR) / "paintingclip_embeddings.safetensors"
CLIP_SENTENCE_IDS = Path(EMBEDDINGS_DIR) / "clip_embeddings_sentence_ids.json"
PAINTINGCLIP_SENTENCE_IDS = Path(EMBEDDINGS_DIR) / "paintingclip_embeddings_sentence_ids.json"
CLIP_EMBEDDINGS_DIR = EMBEDDINGS_DIR
PAINTINGCLIP_EMBEDDINGS_DIR = EMBEDDINGS_DIR

# READ root (repo data - read-only)
PROJECT_ROOT = Path(__file__).resolve().parents[2]
DATA_READ_ROOT = PROJECT_ROOT / "data"

# WRITE root (Space volume - writable)
# HF Spaces uses /data for persistent storage
WRITE_ROOT = Path(os.getenv("HF_HOME", "/data"))

# Check if the directory exists and is writable
if not WRITE_ROOT.exists():
    print(f"⚠️  WRITE_ROOT {WRITE_ROOT} does not exist, trying to create it")
    try:
        WRITE_ROOT.mkdir(parents=True, exist_ok=True)
        print(f"✅ Created WRITE_ROOT: {WRITE_ROOT}")
    except Exception as e:
        print(f"❌ Failed to create {WRITE_ROOT}: {e}")
        raise RuntimeError(f"Cannot create writable directory: {e}")

# Check write permissions
if not os.access(WRITE_ROOT, os.W_OK):
    print(f"❌ WRITE_ROOT {WRITE_ROOT} is not writable")
    print(f"❌ Current permissions: {oct(WRITE_ROOT.stat().st_mode)[-3:]}")
    print(f"❌ Owner: {WRITE_ROOT.owner()}")
    raise RuntimeError(f"Directory {WRITE_ROOT} is not writable")

print(f"✅ Using WRITE_ROOT: {WRITE_ROOT}")
print(f"✅ Using READ_ROOT: {DATA_READ_ROOT}")

# Read-only directories (from repo)
MODELS_DIR = DATA_READ_ROOT / "models"
MARKER_DIR = DATA_READ_ROOT / "marker_output"

# Model directories
PAINTINGCLIP_MODEL_DIR = MODELS_DIR / "PaintingClip"  # Note the capital C

# Writable directories (outside repo)
OUTPUTS_DIR = WRITE_ROOT / "outputs"
ARTIFACTS_DIR = WRITE_ROOT / "artifacts"

# Ensure writable directories exist
for dir_path in [OUTPUTS_DIR, ARTIFACTS_DIR]:
    try:
        dir_path.mkdir(parents=True, exist_ok=True)
        print(f"✅ Ensured directory exists: {dir_path}")
    except Exception as e:
        print(f"⚠️  Could not create directory {dir_path}: {e}")

# Global data variables (will be populated from HF datasets)
sentences: Dict[str, Any] = {}
works: Dict[str, Any] = {}
creators: Dict[str, Any] = {}
topics: Dict[str, Any] = {}
topic_names: Dict[str, Any] = {}

def load_json_from_hf(repo_id: str, filename: str) -> Optional[Dict[str, Any]]:
    """Load a single JSON file from Hugging Face repository"""
    if not HF_HUB_AVAILABLE:
        print(f"⚠️  huggingface_hub not available - cannot load {filename}")
        return None
        
    try:
        print(f"🔍 Downloading {filename} from {repo_id}...")
        file_path = hf_hub_download(
            repo_id=repo_id, 
            filename=filename, 
            repo_type="dataset"
        )
        
        with open(file_path, 'r', encoding='utf-8') as f:
            data = json.load(f)
        
        print(f"✅ Successfully loaded {filename}: {len(data)} entries")
        return data
    except Exception as e:
        print(f"❌ Failed to load {filename} from {repo_id}: {e}")
        return None

def load_json_datasets() -> Optional[Dict[str, Any]]:
    """Load all JSON datasets from Hugging Face"""
    if not HF_HUB_AVAILABLE:
        print("⚠️  huggingface_hub library not available - skipping HF dataset loading")
        return None
        
    try:
        print(" Loading JSON files from Hugging Face repository...")
        
        # Load individual JSON files
        global sentences, works, creators, topics, topic_names
        
        creators = load_json_from_hf(ARTEFACT_JSON_DATASET, 'creators.json') or {}
        sentences = load_json_from_hf(ARTEFACT_JSON_DATASET, 'sentences.json') or {}
        works = load_json_from_hf(ARTEFACT_JSON_DATASET, 'works.json') or {}
        topics = load_json_from_hf(ARTEFACT_JSON_DATASET, 'topics.json') or {}
        topic_names = load_json_from_hf(ARTEFACT_JSON_DATASET, 'topic_names.json') or {}
        
        print(f"✅ Successfully loaded JSON files from HF:")
        print(f"   Sentences: {len(sentences)} entries")
        print(f"   Works: {len(works)} entries")
        print(f"   Creators: {len(creators)} entries")
        print(f"   Topics: {len(topics)} entries")
        print(f"   Topic Names: {len(topic_names)} entries")
        
        return {
            'creators': creators,
            'sentences': sentences,
            'works': works,
            'topics': topics,
            'topic_names': topic_names
        }
    except Exception as e:
        print(f"❌ Failed to load JSON datasets from HF: {e}")
        return None

def load_embeddings_datasets() -> Optional[Dict[str, Any]]:
    """Load embeddings datasets from Hugging Face using direct file download"""
    if not HF_HUB_AVAILABLE:
        print("⚠️  huggingface_hub library not available - skipping HF embeddings loading")
        return None
        
    try:
        print(f" Loading embeddings from {ARTEFACT_EMBEDDINGS_DATASET}...")
        
        # Return a flag indicating we should use direct file download
        # The actual loading will be done in inference.py
        return {
            'use_direct_download': True,
            'repo_id': ARTEFACT_EMBEDDINGS_DATASET
        }
    except Exception as e:
        print(f"❌ Failed to load embeddings datasets from HF: {e}")
        return None

_markdown_dir_cache = None

def clear_markdown_cache() -> bool:
    """Clear the markdown cache to force a fresh download"""
    try:
        import shutil
        markdown_cache_dir = WRITE_ROOT / "markdown_cache"
        if markdown_cache_dir.exists():
            print(f"🗑️  Clearing markdown cache at {markdown_cache_dir}")
            shutil.rmtree(markdown_cache_dir)
            print(f"✅ Markdown cache cleared successfully")
            return True
        else:
            print(f"ℹ️  No markdown cache found to clear")
            return True
    except Exception as e:
        print(f"❌ Failed to clear markdown cache: {e}")
        return False

def get_markdown_cache_info() -> dict:
    """Get information about the current markdown cache"""
    try:
        import shutil
        markdown_cache_dir = WRITE_ROOT / "markdown_cache"
        works_dir = markdown_cache_dir / "works"
        
        if not works_dir.exists():
            return {
                "exists": False,
                "size_gb": 0,
                "work_count": 0,
                "file_count": 0
            }
        
        # Calculate total size
        total_size = sum(f.stat().st_size for f in works_dir.rglob('*') if f.is_file())
        size_gb = total_size / (1024**3)
        
        # Count files and directories
        file_count = len(list(works_dir.rglob('*')))
        work_count = len([d for d in works_dir.iterdir() if d.is_dir()])
        
        return {
            "exists": True,
            "size_gb": round(size_gb, 2),
            "work_count": work_count,
            "file_count": file_count,
            "path": str(works_dir)
        }
    except Exception as e:
        print(f"❌ Failed to get cache info: {e}")
        return {"exists": False, "error": str(e)}

def load_markdown_dataset(force_refresh: bool = False) -> Optional[Path]:
    """Load markdown dataset from Hugging Face and return the local path"""
    if not HF_HUB_AVAILABLE:
        print("⚠️  huggingface_hub not available - cannot load markdown dataset")
        return None
        
    try:
        print(f"�� Loading markdown dataset from {ARTEFACT_MARKDOWN_DATASET}...")
        
        # Create a local cache directory for the markdown dataset
        markdown_cache_dir = WRITE_ROOT / "markdown_cache"
        markdown_cache_dir.mkdir(parents=True, exist_ok=True)
        
        works_dir = markdown_cache_dir / "works"
        
        # Check if we should force refresh or if cache is incomplete
        if force_refresh:
            print("🔄 Force refresh requested - clearing cache")
            clear_markdown_cache()
        else:
            # Check cache completeness
            cache_info = get_markdown_cache_info()
            if cache_info["exists"]:
                print(f"📊 Cache info: {cache_info['work_count']} works, {cache_info['size_gb']}GB")
                
                # If we have significantly fewer works than expected, clear and re-download
                expected_works = 7200  # Based on your dataset
                if cache_info["work_count"] < expected_works * 0.8:  # Less than 80% of expected
                    print(f"⚠️  Cache incomplete ({cache_info['work_count']}/{expected_works} works) - clearing and re-downloading")
                    clear_markdown_cache()
                else:
                    print(f"✅ Using cached markdown dataset at {works_dir}")
                    return works_dir
        
        # Use optimized download approach
        print("📥 Downloading markdown dataset with optimized approach...")
        return _download_markdown_optimized(works_dir)
            from datasets import load_dataset
            print("�� Downloading markdown dataset...")
            # Use huggingface_hub to download files directly instead of datasets library
            from huggingface_hub import list_repo_files
            files = list_repo_files(repo_id=ARTEFACT_MARKDOWN_DATASET, repo_type="dataset")
            
            # Debug: Show dataset structure
            print(f"🔍 Total files in dataset: {len(files)}")
            works_files = [f for f in files if f.startswith("works/")]
            print(f"🔍 Files starting with 'works/': {len(works_files)}")
            if works_files:
                print(f"🔍 Sample work files: {works_files[:5]}")
            
            # Filter for work directories and files
            work_dirs = set()
            for file_path in files:
                if file_path.startswith("works/"):
                    parts = file_path.split("/")
                    if len(parts) >= 2:
                        work_id = parts[1]
                        if work_id.startswith("W"):  # Only include work IDs
                            work_dirs.add(work_id)
            
            print(f" Found {len(work_dirs)} work directories to download")
            
            # Debug: Show sample work IDs
            work_list = sorted(list(work_dirs))
            print(f"🔍 Sample work IDs: {work_list[:10]}")
            print(f"🔍 Last few work IDs: {work_list[-5:]}")
            
            # Download each work directory
            for i, work_id in enumerate(work_dirs):
                if i % 100 == 0:
                    print(f" Downloaded {i}/{len(work_dirs)} work directories...")
                    if i < 10:  # Show first 10 work IDs being processed
                        print(f"🔍 Processing work: {work_id}")
                
                work_dir = works_dir / work_id
                work_dir.mkdir(parents=True, exist_ok=True)
                
                # Download markdown file
                try:
                    md_file = hf_hub_download(
                        repo_id=ARTEFACT_MARKDOWN_DATASET,
                        filename=f"works/{work_id}/{work_id}.md",
                        repo_type="dataset"
                    )
                    # Copy to our cache
                    import shutil
                    shutil.copy2(md_file, work_dir / f"{work_id}.md")
                    if i < 5:  # Debug: Show first few successful downloads
                        print(f"✅ Downloaded markdown for {work_id}")
                except Exception as e:
                    print(f"⚠️  Could not download markdown for {work_id}: {e}")
                
                # Download images
                try:
                    images_dir = work_dir / "images"
                    images_dir.mkdir(exist_ok=True)
                    
                    # Get list of image files for this work
                    work_files = [f for f in files if f.startswith(f"works/{work_id}/images/")]
                    
                    if i < 3:  # Debug: Show image count for first few works
                        print(f"🔍 Found {len(work_files)} images for {work_id}")
                    
                    for img_file in work_files:
                        try:
                            downloaded_file = hf_hub_download(
                                repo_id=ARTEFACT_MARKDOWN_DATASET,
                                filename=img_file,
                                repo_type="dataset"
                            )
                            # Copy to our cache
                            img_name = img_file.split("/")[-1]
                            shutil.copy2(downloaded_file, images_dir / img_name)
                        except Exception as e:
                            print(f"⚠️  Could not download image {img_file}: {e}")
                            
                except Exception as e:
                    print(f"⚠️  Could not download images for {work_id}: {e}")
            
            print(f"✅ Successfully downloaded markdown dataset to {works_dir}")
            return works_dir
            
        else:
            print("⚠️  datasets library not available - using fallback method")
            # Fallback: try to download individual files
            return _download_markdown_files_fallback(markdown_cache_dir)
            
    except Exception as e:
        print(f"❌ Failed to load markdown dataset: {e}")
        return None

def _download_markdown_optimized(works_dir: Path) -> Optional[Path]:
    """Optimized markdown dataset download with parallel processing"""
    try:
        from huggingface_hub import list_repo_files
        import concurrent.futures
        import threading
        import time
        
        # Get the list of files in the dataset
        print("🔍 Discovering files in dataset...")
        files = list_repo_files(repo_id=ARTEFACT_MARKDOWN_DATASET, repo_type="dataset")
        
        # Filter for work directories
        work_dirs = set()
        for file_path in files:
            if file_path.startswith("works/"):
                parts = file_path.split("/")
                if len(parts) >= 2:
                    work_id = parts[1]
                    if work_id.startswith("W"):  # Only include work IDs
                        work_dirs.add(work_id)
        
        print(f" Found {len(work_dirs)} work directories to download")
        
        # Phase 1: Download only markdown files (fast)
        print("📄 Phase 1: Downloading markdown files only...")
        _download_markdown_files_parallel(works_dir, work_dirs, files)
        
        # Phase 2: Download images in batches (slower but manageable)
        print("🖼️  Phase 2: Downloading images in batches...")
        _download_images_batch(works_dir, work_dirs, files)
        
        print(f"✅ Successfully downloaded markdown dataset to {works_dir}")
        return works_dir
        
    except Exception as e:
        print(f"❌ Optimized download failed: {e}")
        return None

def _download_markdown_files_parallel(works_dir: Path, work_dirs: set, files: list) -> None:
    """Download markdown files in parallel for speed"""
    import concurrent.futures
    import threading
    import time
    
    def download_markdown_file(work_id: str) -> bool:
        """Download a single markdown file"""
        try:
            work_dir = works_dir / work_id
            work_dir.mkdir(parents=True, exist_ok=True)
            
            md_file = hf_hub_download(
                repo_id=ARTEFACT_MARKDOWN_DATASET,
                filename=f"works/{work_id}/{work_id}.md",
                repo_type="dataset"
            )
            
            import shutil
            shutil.copy2(md_file, work_dir / f"{work_id}.md")
            return True
        except Exception as e:
            print(f"⚠️  Could not download markdown for {work_id}: {e}")
            return False
    
    # Download markdown files in parallel
    work_list = list(work_dirs)
    completed = 0
    failed = 0
    
    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
        future_to_work = {executor.submit(download_markdown_file, work_id): work_id for work_id in work_list}
        
        for future in concurrent.futures.as_completed(future_to_work):
            work_id = future_to_work[future]
            try:
                success = future.result()
                if success:
                    completed += 1
                else:
                    failed += 1
                
                if (completed + failed) % 500 == 0:
                    print(f"📄 Downloaded {completed}/{len(work_list)} markdown files (failed: {failed})")
                    
            except Exception as e:
                print(f"❌ Error processing {work_id}: {e}")
                failed += 1
    
    print(f"✅ Phase 1 complete: {completed} markdown files downloaded, {failed} failed")

def _download_images_batch(works_dir: Path, work_dirs: set, files: list) -> None:
    """Download images in batches to avoid overwhelming the server"""
    import concurrent.futures
    import time
    
    def download_work_images(work_id: str) -> tuple:
        """Download all images for a single work"""
        try:
            work_dir = works_dir / work_id
            images_dir = work_dir / "images"
            images_dir.mkdir(exist_ok=True)
            
            # Get list of image files for this work
            work_files = [f for f in files if f.startswith(f"works/{work_id}/images/")]
            
            downloaded = 0
            failed = 0
            
            for img_file in work_files:
                try:
                    downloaded_file = hf_hub_download(
                        repo_id=ARTEFACT_MARKDOWN_DATASET,
                        filename=img_file,
                        repo_type="dataset"
                    )
                    
                    import shutil
                    img_name = img_file.split("/")[-1]
                    shutil.copy2(downloaded_file, images_dir / img_name)
                    downloaded += 1
                    
                except Exception as e:
                    failed += 1
                    # Don't print every single image error to avoid spam
                    if failed <= 3:  # Only print first few errors
                        print(f"⚠️  Could not download image {img_file}: {e}")
            
            return (work_id, downloaded, failed)
            
        except Exception as e:
            print(f"❌ Error downloading images for {work_id}: {e}")
            return (work_id, 0, 1)
    
    # Process works in batches to avoid overwhelming the server
    work_list = list(work_dirs)
    batch_size = 50  # Process 50 works at a time
    total_downloaded = 0
    total_failed = 0
    
    for i in range(0, len(work_list), batch_size):
        batch = work_list[i:i + batch_size]
        print(f"🖼️  Processing image batch {i//batch_size + 1}/{(len(work_list) + batch_size - 1)//batch_size} ({len(batch)} works)")
        
        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
            future_to_work = {executor.submit(download_work_images, work_id): work_id for work_id in batch}
            
            for future in concurrent.futures.as_completed(future_to_work):
                work_id = future_to_work[future]
                try:
                    work_id, downloaded, failed = future.result()
                    total_downloaded += downloaded
                    total_failed += failed
                except Exception as e:
                    print(f"❌ Error processing {work_id}: {e}")
                    total_failed += 1
        
        # Small delay between batches to be nice to the server
        time.sleep(1)
    
    print(f"✅ Phase 2 complete: {total_downloaded} images downloaded, {total_failed} failed")

def _download_markdown_files_fallback(cache_dir: Path) -> Optional[Path]:
    """Fallback method to download markdown files individually"""
    try:
        works_dir = cache_dir / "works"
        works_dir.mkdir(exist_ok=True)
        
        # This is a simplified fallback - you might need to implement
        # a more sophisticated file discovery mechanism
        print("⚠️  Using fallback markdown loading - some files may be missing")
        return works_dir
        
    except Exception as e:
        print(f"❌ Fallback markdown loading failed: {e}")
        return None

def get_markdown_dir(force_refresh: bool = False) -> Path:
    """Get the markdown directory, loading from HF if needed"""
    global _markdown_dir_cache
    
    if _markdown_dir_cache is None or force_refresh:
        _markdown_dir_cache = load_markdown_dataset(force_refresh=force_refresh)
    
    if _markdown_dir_cache and _markdown_dir_cache.exists():
        return _markdown_dir_cache
    else:
        # Fallback to local directory if HF loading fails
        print("⚠️  Using fallback local markdown directory")
        return DATA_READ_ROOT / "marker_output"

# Initialize datasets
JSON_DATASETS = load_json_datasets()
EMBEDDINGS_DATASETS = load_embeddings_datasets()

# Initialize data loading
if JSON_DATASETS is None:
    print("⚠️  Some data failed to load from HF datasets")
else:
    print("✅ All data loaded successfully from HF datasets")

# Add this function for backward compatibility
def st_load_file(file_path: Path) -> Any:
    """Load a file using safetensors or other methods"""
    try:
        if file_path.suffix == '.safetensors':
            import safetensors
            return safetensors.safe_open(str(file_path), framework="pt")
        else:
            import torch
            return torch.load(str(file_path))
    except ImportError:
        print(f"⚠️  Required library not available for loading {file_path}")
        return None
    except Exception as e:
        print(f"❌ Error loading {file_path}: {e}")
        return None