Fix pyarrow compatibility issue with datasets library

- Update datasets to 2.16.1 and pyarrow to 14.0.2 for compatibility
- Add graceful error handling when dataset can't load
- Return 503 error instead of crashing when dataset unavailable
- Maintain research integrity - no mock data fallback

🤖 Generated with Claude Code

Co-Authored-By: Claude <noreply@anthropic.com>

backend/model_service.py

@@ -1178,6 +1178,14 @@ async def get_swe_bench_tasks(
         # Try to load dataset if not already loaded
         await swe_bench_service.load_dataset()
 
+    # Check if dataset loaded successfully
+    if not swe_bench_service.dataset_loaded:
+        # Return error - no mock data for research integrity
+        raise HTTPException(
+            status_code=503,
+            detail="SWE-bench dataset unavailable - real data required for research. Check server logs for details."
+        )
+
     tasks = swe_bench_service.get_tasks(
         category=category,
         difficulty=difficulty,

backend/swe_bench_service.py

@@ -101,7 +101,19 @@ class SWEBenchService:
     async def load_dataset(self, dataset_name: str = "princeton-nlp/SWE-bench_Lite"):
         """Load SWE-bench dataset from Hugging Face"""
         try:
-            from datasets import load_dataset
+            # Check if datasets library is available with proper dependencies
+            try:
+                from datasets import load_dataset
+                import pyarrow as pa
+                # Verify pyarrow has the required attribute
+                if not hasattr(pa, 'PyExtensionType'):
+                    logger.error("pyarrow version incompatible with datasets library")
+                    self.dataset_loaded = False
+                    return
+            except ImportError as ie:
+                logger.error(f"Required libraries not properly installed: {ie}")
+                self.dataset_loaded = False
+                return
 
             logger.info(f"Loading SWE-bench dataset: {dataset_name}")
 
@@ -129,19 +141,17 @@ class SWEBenchService:
                 self.dataset_loaded = True
                 logger.info(f"Loaded {len(self.tasks)} SWE-bench tasks")
             except Exception as dataset_error:
-                logger.error(f"Could not load full dataset: {dataset_error}")
+                logger.error(f"Could not load dataset: {dataset_error}")
                 # No mock data - research requires real dataset
-                raise Exception("SWE-bench dataset unavailable - real data required for research")
+                self.dataset_loaded = False
+                return
 
             # Initialize metrics cache
             self._update_metrics_cache()
 
-        except ImportError:
-            logger.error("datasets library not installed - real data required")
-            raise ImportError("datasets library required for SWE-bench - pip install datasets")
         except Exception as e:
             logger.error(f"Failed to load SWE-bench dataset: {e}")
-            raise Exception(f"SWE-bench dataset loading failed: {e}")
+            self.dataset_loaded = False
 
     def get_tasks(
         self,

requirements.txt

@@ -16,4 +16,5 @@ aiofiles==23.2.1
 python-dotenv==1.0.0
 
 # SWE-bench support
-datasets==2.14.0
+datasets==2.16.1
+pyarrow==14.0.2