Try to fix markdown loading

backend/runner/config.py

@@ -202,19 +202,36 @@ def load_markdown_dataset() -> Optional[Path]:
             from huggingface_hub import list_repo_files
             files = list_repo_files(repo_id=ARTEFACT_MARKDOWN_DATASET, repo_type="dataset")
             
+            # Debug: Show dataset structure
+            print(f"🔍 Total files in dataset: {len(files)}")
+            works_files = [f for f in files if f.startswith("works/")]
+            print(f"🔍 Files starting with 'works/': {len(works_files)}")
+            if works_files:
+                print(f"🔍 Sample work files: {works_files[:5]}")
+            
             # Filter for work directories and files
             work_dirs = set()
             for file_path in files:
-                if file_path.startswith("works/") and "/" in file_path[7:]:
-                    work_id = file_path.split("/")[1]
-                    work_dirs.add(work_id)
+                if file_path.startswith("works/"):
+                    parts = file_path.split("/")
+                    if len(parts) >= 2:
+                        work_id = parts[1]
+                        if work_id.startswith("W"):  # Only include work IDs
+                            work_dirs.add(work_id)
             
             print(f" Found {len(work_dirs)} work directories to download")
             
+            # Debug: Show sample work IDs
+            work_list = sorted(list(work_dirs))
+            print(f"🔍 Sample work IDs: {work_list[:10]}")
+            print(f"🔍 Last few work IDs: {work_list[-5:]}")
+            
             # Download each work directory
             for i, work_id in enumerate(work_dirs):
                 if i % 100 == 0:
                     print(f" Downloaded {i}/{len(work_dirs)} work directories...")
+                    if i < 10:  # Show first 10 work IDs being processed
+                        print(f"🔍 Processing work: {work_id}")
                 
                 work_dir = works_dir / work_id
                 work_dir.mkdir(parents=True, exist_ok=True)
@@ -229,6 +246,8 @@ def load_markdown_dataset() -> Optional[Path]:
                     # Copy to our cache
                     import shutil
                     shutil.copy2(md_file, work_dir / f"{work_id}.md")
+                    if i < 5:  # Debug: Show first few successful downloads
+                        print(f"✅ Downloaded markdown for {work_id}")
                 except Exception as e:
                     print(f"⚠️  Could not download markdown for {work_id}: {e}")
                 
@@ -240,6 +259,9 @@ def load_markdown_dataset() -> Optional[Path]:
                     # Get list of image files for this work
                     work_files = [f for f in files if f.startswith(f"works/{work_id}/images/")]
                     
+                    if i < 3:  # Debug: Show image count for first few works
+                        print(f"🔍 Found {len(work_files)} images for {work_id}")
+                    
                     for img_file in work_files:
                         try:
                             downloaded_file = hf_hub_download(