Fix magic-pdf installation and remove fallbacks, ensuring proper operation in Hugging Face Spaces

Dockerfile

@@ -9,6 +9,8 @@ RUN apt-get update && \
     libglib2.0-0 \
     wget \
     git \
+    libopenblas-dev \
+    libgomp1 \
     && rm -rf /var/lib/apt/lists/*
 
 # Set working directory
@@ -18,34 +20,44 @@ WORKDIR /app
 ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     UPLOAD_FOLDER=/tmp/pdf_uploads \
-    OUTPUT_FOLDER=/tmp/pdf_output
+    OUTPUT_FOLDER=/tmp/pdf_output \
+    HF_HUB_DISABLE_PROGRESS_BARS=1 \
+    HF_HUB_ENABLE_HF_TRANSFER=0
 
 # Create necessary directories and set permissions
-RUN mkdir -p /tmp/pdf_uploads /tmp/pdf_output /tmp/models \
+RUN mkdir -p /tmp/pdf_uploads /tmp/pdf_output \
+    && mkdir -p /tmp/models/MFD/YOLO \
+    && mkdir -p /tmp/models/MFR/unimernet \
+    && mkdir -p /tmp/models/table/rapid \
+    && mkdir -p /tmp/models/layout/doclayout \
     && chmod -R 777 /tmp/pdf_uploads /tmp/pdf_output /tmp/models
 
+# Create magic-pdf config directories
+RUN mkdir -p /root/.config/magic_pdf
+
 # Copy requirements and app files
 COPY requirements.txt /app/
 COPY *.py /app/
 COPY entrypoint.sh /app/
 RUN chmod +x /app/entrypoint.sh
 
-# Create a non-root user and change ownership
-RUN useradd -m -u 1000 appuser && \
-    chown -R appuser:appuser /app /tmp/pdf_uploads /tmp/pdf_output /tmp/models
-
 # Install Python dependencies
-RUN pip install --no-cache-dir -r requirements.txt && \
-    pip install --no-cache-dir pymupdf
+RUN pip install --no-cache-dir --upgrade pip setuptools wheel
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy default magic-pdf config
+RUN echo '{"device-mode":"cpu","layout-config":{"model":"doclayout_yolo","enable":true},"formula-config":{"mfd_model":"yolo_v8_mfd","mfr_model":"unimernet_small","enable":false},"table-config":{"model":"rapid_table","sub_model":"slanet_plus","enable":false}}' > /root/.config/magic_pdf/magic-pdf.json
 
-# Switch to non-root user
-USER appuser
+# Download model files directly
+RUN curl -L https://huggingface.co/marcosremar2/mineru-models/resolve/main/doclayout_yolo.pt -o /tmp/models/layout/doclayout/doclayout_yolo.pt
+RUN curl -L https://huggingface.co/marcosremar2/mineru-models/resolve/main/yolo_v8_ft.pt -o /tmp/models/MFD/YOLO/yolo_v8_ft.pt
 
-# Install MinerU and PyMuPDF (fallback)
-RUN pip install --no-cache-dir --user minerupdf
+# Verify model files were downloaded correctly
+RUN ls -la /tmp/models/layout/doclayout/doclayout_yolo.pt
+RUN ls -la /tmp/models/MFD/YOLO/yolo_v8_ft.pt
 
 # Test the installation
-RUN ~/.local/bin/magic-pdf --version || echo "magic-pdf not working, will fallback to PyMuPDF"
+RUN magic-pdf --version
 
 # Expose port
 EXPOSE 7860

app.py

@@ -26,39 +26,61 @@ logging.basicConfig(
 logger = logging.getLogger()
 
 # Constants
-UPLOAD_FOLDER = os.environ.get('UPLOAD_FOLDER', '/tmp/pdf_uploads')
-OUTPUT_FOLDER = os.environ.get('OUTPUT_FOLDER', '/tmp/pdf_output')
+UPLOAD_FOLDER = '/tmp/pdf_uploads'
+OUTPUT_FOLDER = '/tmp/pdf_output'
 ALLOWED_EXTENSIONS = {'pdf'}
 MAX_FILE_SIZE = 50 * 1024 * 1024  # 50MB
 
-# Ensure the directories exist and are writable
+# Ensure the directories exist
+os.makedirs(UPLOAD_FOLDER, exist_ok=True)
+os.makedirs(OUTPUT_FOLDER, exist_ok=True)
+logging.info(f"Using upload folder: {UPLOAD_FOLDER}")
+logging.info(f"Using output folder: {OUTPUT_FOLDER}")
+
+# Create config directories and files if they don't exist
+CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".config", "magic_pdf")
+os.makedirs(CONFIG_DIR, exist_ok=True)
+CONFIG_FILE = os.path.join(CONFIG_DIR, "magic-pdf.json")
+
+# Default configuration
+DEFAULT_CONFIG = {
+    "device-mode": "cpu",
+    "layout-config": {
+        "model": "doclayout_yolo",
+        "enable": True
+    },
+    "formula-config": {
+        "mfd_model": "yolo_v8_mfd",
+        "mfr_model": "unimernet_small",
+        "enable": False
+    },
+    "table-config": {
+        "model": "rapid_table",
+        "sub_model": "slanet_plus",
+        "enable": False
+    }
+}
+
+# Write config file if it doesn't exist
+if not os.path.exists(CONFIG_FILE):
+    with open(CONFIG_FILE, 'w') as f:
+        json.dump(DEFAULT_CONFIG, f, indent=2)
+    logging.info(f"Created magic-pdf config at {CONFIG_FILE}")
+
+# Also create the config in the home directory as a fallback
+HOME_CONFIG = os.path.join(os.path.expanduser("~"), "magic-pdf.json")
+if not os.path.exists(HOME_CONFIG):
+    with open(HOME_CONFIG, 'w') as f:
+        json.dump(DEFAULT_CONFIG, f, indent=2)
+    logging.info(f"Created magic-pdf config at {HOME_CONFIG}")
+
+# Verify magic-pdf installation
 try:
-    os.makedirs(UPLOAD_FOLDER, exist_ok=True)
-    os.makedirs(OUTPUT_FOLDER, exist_ok=True)
-    # Test if we can write to these directories
-    test_upload_file = os.path.join(UPLOAD_FOLDER, '.test_file')
-    test_output_file = os.path.join(OUTPUT_FOLDER, '.test_file')
-    
-    with open(test_upload_file, 'w') as f:
-        f.write('test')
-    os.remove(test_upload_file)
-    
-    with open(test_output_file, 'w') as f:
-        f.write('test')
-    os.remove(test_output_file)
-    
-    logging.info(f"Using upload folder: {UPLOAD_FOLDER}")
-    logging.info(f"Using output folder: {OUTPUT_FOLDER}")
+    result = subprocess.run(['magic-pdf', '--version'], capture_output=True, text=True)
+    logging.info(f"magic-pdf version: {result.stdout.strip()}")
 except Exception as e:
-    # Fall back to user's home directory
-    logging.warning(f"Cannot use specified directories: {str(e)}")
-    USER_HOME = os.path.expanduser("~")
-    UPLOAD_FOLDER = os.path.join(USER_HOME, 'pdf_uploads')
-    OUTPUT_FOLDER = os.path.join(USER_HOME, 'pdf_output')
-    os.makedirs(UPLOAD_FOLDER, exist_ok=True)
-    os.makedirs(OUTPUT_FOLDER, exist_ok=True)
-    logging.info(f"Using fallback upload folder: {UPLOAD_FOLDER}")
-    logging.info(f"Using fallback output folder: {OUTPUT_FOLDER}")
+    logging.error(f"Error checking magic-pdf: {str(e)}")
+    raise RuntimeError("magic-pdf is not installed properly. Please check installation.")
 
 # Function to check if file extension is allowed
 def allowed_file(filename):
@@ -364,220 +386,128 @@ def help_output():
 # Route for PDF conversion
 @app.route('/api/convert', methods=['POST'])
 def convert_pdf():
-    if 'file' not in request.files:
-        return jsonify({"error": "No file part in the request"}), 400
-    
-    file = request.files['file']
-    
-    if file.filename == '':
-        return jsonify({"error": "No file selected"}), 400
-    
-    if not allowed_file(file.filename):
-        return jsonify({"error": f"Only {', '.join(ALLOWED_EXTENSIONS)} files are allowed"}), 400
-    
+    """
+    API endpoint for PDF conversion.
+    """
     try:
-        # Create a unique session ID
+        # Check if file is in the request
+        if 'file' not in request.files:
+            return jsonify({
+                "success": False,
+                "error": "No file part in the request"
+            }), 400
+            
+        file = request.files['file']
+        
+        # Check if file is selected
+        if file.filename == '':
+            return jsonify({
+                "success": False,
+                "error": "No file selected"
+            }), 400
+            
+        # Check if file is a PDF
+        if not allowed_file(file.filename):
+            return jsonify({
+                "success": False,
+                "error": "File must be a PDF"
+            }), 400
+            
+        # Generate session ID
         session_id = str(uuid.uuid4())
         session_dir = os.path.join(OUTPUT_FOLDER, session_id)
-        
-        try:
-            os.makedirs(session_dir, exist_ok=True)
-            logging.info(f"Created session directory: {session_dir}")
-        except PermissionError as e:
-            logging.error(f"Permission error creating directory {session_dir}: {str(e)}")
-            # Try using a temp directory instead
-            session_dir = tempfile.mkdtemp(prefix="minerupdf_")
-            logging.info(f"Using temporary directory instead: {session_dir}")
+        os.makedirs(session_dir, exist_ok=True)
         
         # Save the uploaded file
-        input_path = os.path.join(UPLOAD_FOLDER, f"{session_id}_{file.filename}")
-        try:
-            file.save(input_path)
-            logging.info(f"Saved uploaded file to {input_path}")
-        except PermissionError:
-            # Try saving in the temp directory
-            input_path = os.path.join(session_dir, f"{session_id}_{file.filename}")
-            file.save(input_path)
-            logging.info(f"Saved uploaded file to alternate location: {input_path}")
+        filename = os.path.basename(file.filename)
+        base_filename = os.path.splitext(filename)[0]
+        input_path = os.path.join(session_dir, filename)
+        file.save(input_path)
         
-        # Get output file paths
-        base_filename = os.path.splitext(os.path.basename(file.filename))[0]
-        base_filename = ''.join(c if c.isalnum() or c in ['_', '-', '.'] else '_' for c in base_filename)
+        logging.info(f"Saved PDF file to {input_path}")
         
+        # Set paths for output files
         markdown_path = os.path.join(session_dir, f"{base_filename}.md")
         json_path = os.path.join(session_dir, f"{base_filename}.json")
         
-        # Define alternate locations to search for generated files
-        alternate_locations = [
-            os.path.dirname(input_path),  # Check PDF directory
-            os.path.join(OUTPUT_FOLDER, base_filename),  # Check output folder with base filename
-            session_dir  # Default session directory
-        ]
-        
         # URLs for file download
         markdown_url = f"/download/{session_id}/{base_filename}.md"
         json_url = f"/download/{session_id}/{base_filename}.json"
         
-        try:
-            # Run MinerU to convert the PDF
-            cmd = [
-                "magic-pdf", 
-                "--path", input_path, 
-                "--output-dir", session_dir
-            ]
-            
-            logging.info(f"Running command: {' '.join(cmd)}")
-            process = subprocess.run(cmd, capture_output=True, text=True, check=False)
-            
-            if process.returncode != 0:
-                logging.error(f"MinerU conversion failed: {process.stderr}")
-                
-                # Fallback to PyMuPDF conversion
-                try:
-                    import fitz  # PyMuPDF
-                    
-                    logging.info(f"Falling back to PyMuPDF for PDF conversion: {input_path}")
-                    # Open the PDF file
-                    pdf = fitz.open(input_path)
-                    
-                    # Extract text and create markdown
-                    markdown_content = f"# {base_filename}\n\n"
-                    json_content = {
-                        "title": base_filename,
-                        "pages": len(pdf),
-                        "sections": []
-                    }
-                    
-                    for page_num in range(len(pdf)):
-                        page = pdf[page_num]
-                        text = page.get_text()
-                        
-                        # Add to markdown
-                        markdown_content += f"## Page {page_num + 1}\n\n{text}\n\n"
-                        
-                        # Add to JSON
-                        json_content["sections"].append({
-                            "title": f"Page {page_num + 1}",
-                            "content": text
-                        })
-                    
-                    # Save markdown
-                    with open(markdown_path, 'w', encoding='utf-8') as f:
-                        f.write(markdown_content)
-                    
-                    # Save JSON
-                    with open(json_path, 'w', encoding='utf-8') as f:
-                        json.dump(json_content, f, indent=2, ensure_ascii=False)
-                    
-                    logging.info(f"Successfully converted PDF to markdown and JSON using PyMuPDF")
-                except ImportError:
-                    # PyMuPDF not available
-                    return jsonify({
-                        "error": "PDF conversion failed with both MinerU and PyMuPDF",
-                        "details": process.stderr,
-                        "command": " ".join(cmd)
-                    }), 500
-            
-            # Check if files were generated
-            md_exists = os.path.exists(markdown_path)
-            json_exists = os.path.exists(json_path)
-            
-            # Search in alternate locations if files not found
-            if not md_exists or not json_exists:
-                logging.info("Files not found in primary location, searching alternate locations...")
-                
-                for location in alternate_locations:
-                    if not md_exists:
-                        alt_md_path = os.path.join(location, f"{base_filename}.md")
-                        if os.path.exists(alt_md_path):
-                            # Copy to session directory for download access
-                            try:
-                                shutil.copy(alt_md_path, markdown_path)
-                                md_exists = True
-                                logging.info(f"Found and copied Markdown file from {alt_md_path}")
-                            except:
-                                # If copy fails, update path to point to original
-                                markdown_path = alt_md_path
-                                md_exists = True
-                                logging.info(f"Found Markdown file at alternate location: {alt_md_path}")
-                    
-                    if not json_exists:
-                        alt_json_path = os.path.join(location, f"{base_filename}.json")
-                        if os.path.exists(alt_json_path):
-                            # Copy to session directory for download access
-                            try:
-                                shutil.copy(alt_json_path, json_path)
-                                json_exists = True
-                                logging.info(f"Found and copied JSON file from {alt_json_path}")
-                            except:
-                                # If copy fails, update path to point to original
-                                json_path = alt_json_path
-                                json_exists = True
-                                logging.info(f"Found JSON file at alternate location: {alt_json_path}")
-            
-            # If still not found, check for any markdown or JSON files in the directory
-            if not md_exists:
-                md_files = [f for f in os.listdir(session_dir) if f.endswith('.md')]
-                if md_files:
-                    markdown_path = os.path.join(session_dir, md_files[0])
-                    md_exists = True
-                    logging.info(f"Found Markdown file at alternate location: {markdown_path}")
-            
-            if not json_exists:
-                json_files = [f for f in os.listdir(session_dir) if f.endswith('.json')]
-                if json_files:
-                    json_path = os.path.join(session_dir, json_files[0])
-                    json_exists = True
-                    logging.info(f"Found JSON file at alternate location: {json_path}")
-            
-            if md_exists:
-                with open(markdown_path, 'r', encoding='utf-8') as f:
-                    markdown_content = f.read()
-            else:
-                markdown_content = None
-                logging.warning(f"Markdown file not found at {markdown_path}")
-            
-            if json_exists:
-                with open(json_path, 'r', encoding='utf-8') as f:
-                    json_content = json.load(f)
-            else:
-                json_content = None
-                logging.warning(f"JSON file not found at {json_path}")
-            
-            return jsonify({
-                "success": True,
-                "session_id": session_id,
-                "markdown_url": markdown_url if md_exists else None,
-                "json_url": json_url if json_exists else None,
-                "markdown_content": markdown_content,
-                "json_content": json_content,
-                "message": "PDF conversion completed"
-            })
-            
-        except Exception as e:
-            logging.error(f"Error during conversion: {str(e)}")
-            logging.error(traceback.format_exc())
+        # Run magic-pdf to convert the PDF
+        command = ["magic-pdf", "--path", input_path, "--output-dir", session_dir]
+        
+        logging.info(f"Running command: {' '.join(command)}")
+        
+        # Execute the command and capture output
+        process_log = []
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True
+        )
+        
+        # Capture output in real-time
+        for line in process.stdout:
+            process_log.append(line)
+            logging.info(f"magic-pdf output: {line.strip()}")
+        
+        # Wait for process to complete
+        return_code = process.wait()
+        
+        if return_code != 0:
+            error_message = "\n".join(process_log)
+            logging.error(f"magic-pdf failed with code {return_code}")
+            logging.error(f"Output: {error_message}")
             
             return jsonify({
-                "error": "Error processing PDF file",
-                "details": str(e)
+                "success": False,
+                "error": f"PDF conversion failed with exit code {return_code}",
+                "log": error_message
             }), 500
-            
-    except Exception as e:
-        logging.error(f"General error: {str(e)}")
-        logging.error(traceback.format_exc())
         
+        # Check if files were generated
+        md_exists = os.path.exists(markdown_path)
+        json_exists = os.path.exists(json_path)
+        
+        logging.info(f"Markdown file exists: {md_exists} at {markdown_path}")
+        logging.info(f"JSON file exists: {json_exists} at {json_path}")
+        
+        # Read file contents if they exist
+        if md_exists:
+            with open(markdown_path, 'r', encoding='utf-8') as f:
+                markdown_content = f.read()
+        else:
+            markdown_content = None
+            logging.warning(f"Markdown file not found at {markdown_path}")
+        
+        if json_exists:
+            with open(json_path, 'r', encoding='utf-8') as f:
+                json_content = json.load(f)
+        else:
+            json_content = None
+            logging.warning(f"JSON file not found at {json_path}")
+        
+        # Return results
+        return jsonify({
+            "success": True,
+            "session_id": session_id,
+            "markdown_url": markdown_url if md_exists else None,
+            "json_url": json_url if json_exists else None,
+            "markdown_content": markdown_content,
+            "json_content": json_content,
+            "input_file": filename,
+            "log": "\n".join(process_log)
+        })
+        
+    except Exception as e:
+        logging.exception("Error in convert_pdf endpoint")
         return jsonify({
-            "error": "Failed to process request",
-            "details": str(e)
+            "success": False,
+            "error": str(e),
+            "traceback": traceback.format_exc()
         }), 500
-    finally:
-        # Clean up the input file
-        try:
-            if 'input_path' in locals() and os.path.exists(input_path):
-                os.remove(input_path)
-        except Exception as e:
-            logging.warning(f"Failed to clean up input file: {str(e)}")
 
 # Route to download converted files
 @app.route('/download/<session_id>/<filename>')

entrypoint.sh

@@ -8,39 +8,25 @@ echo "Current directory: $(pwd)"
 echo "Directory listing:"
 ls -la
 
-# Create necessary directories
-mkdir -p /tmp/pdf_uploads
-mkdir -p /tmp/pdf_output
-mkdir -p /tmp/models/MFD/YOLO
-mkdir -p /tmp/models/MFR/unimernet
-mkdir -p /tmp/models/table/rapid
-mkdir -p /tmp/models/layout/doclayout
+# Set environment variables to disable HF progress bars (prevents hanging)
+export HF_HUB_DISABLE_PROGRESS_BARS=1
+export HF_HUB_ENABLE_HF_TRANSFER=0
 
-# Set permissions (ensure directories are writable)
-chmod -R 777 /tmp/pdf_uploads
-chmod -R 777 /tmp/pdf_output
-chmod -R 777 /tmp/models
-
-# Check if magic-pdf command exists
-if ! command -v magic-pdf &> /dev/null; then
-    echo "Warning: magic-pdf command not found. Installing MinerU PDF processing tools..."
-    pip install --no-cache-dir minerupdf
-    
-    # Verify installation
-    if command -v magic-pdf &> /dev/null; then
-        echo "Successfully installed magic-pdf."
-    else
-        echo "Failed to install magic-pdf. Will use PyMuPDF fallback."
-        pip install --no-cache-dir pymupdf
+# Verify directories existence and permissions
+echo "Checking directories..."
+for dir in /tmp/pdf_uploads /tmp/pdf_output /tmp/models/MFD/YOLO /tmp/models/MFR/unimernet /tmp/models/table/rapid /tmp/models/layout/doclayout
+do
+    if [ ! -d "$dir" ]; then
+        echo "Creating directory: $dir"
+        mkdir -p "$dir"
     fi
-else
-    echo "magic-pdf command is already installed."
-fi
+    chmod -R 777 "$dir"
+    echo "Directory $dir is ready"
+done
 
-# Configure magic-pdf
-mkdir -p ~/.config/magic_pdf
-cat > ~/.config/magic_pdf/magic-pdf.json << EOL
-{
+# Copy config file to all possible locations to ensure it's found
+echo "Setting up magic-pdf config..."
+CONFIG_CONTENT='{
   "device-mode": "cpu",
   "layout-config": {
     "model": "doclayout_yolo",
@@ -49,61 +35,84 @@ cat > ~/.config/magic_pdf/magic-pdf.json << EOL
   "formula-config": {
     "mfd_model": "yolo_v8_mfd",
     "mfr_model": "unimernet_small",
-    "enable": true
+    "enable": false
   },
   "table-config": {
     "model": "rapid_table",
     "sub_model": "slanet_plus",
-    "enable": true,
-    "max_time": 400
+    "enable": false
   }
-}
-EOL
+}'
 
-# Download required model files if they don't exist or are empty
-function download_model() {
+# Create config in all possible locations
+mkdir -p ~/.config/magic_pdf
+echo "$CONFIG_CONTENT" > ~/.config/magic_pdf/magic-pdf.json
+echo "$CONFIG_CONTENT" > ~/magic-pdf.json
+echo "$CONFIG_CONTENT" > /app/magic-pdf.json
+echo "$CONFIG_CONTENT" > /home/user/magic-pdf.json
+echo "$CONFIG_CONTENT" > /root/.config/magic_pdf/magic-pdf.json
+
+# Download model function with validation
+function download_model_with_validation() {
     local model_path=$1
     local model_url=$2
     local max_attempts=3
     local attempt=1
     
+    echo "Checking model file: $model_path"
+    
     if [ ! -f "$model_path" ] || [ ! -s "$model_path" ]; then
-        echo "Downloading model to $model_path from $model_url"
         while [ $attempt -le $max_attempts ]; do
-            echo "Download attempt $attempt of $max_attempts..."
-            curl -L "$model_url" --output "$model_path" --retry 3 --retry-delay 2
+            echo "Downloading model attempt $attempt/$max_attempts: $model_url"
+            curl -L --retry 5 --retry-delay 2 "$model_url" -o "$model_path"
             
+            # Verify file exists and has content
             if [ -f "$model_path" ] && [ -s "$model_path" ]; then
-                echo "Successfully downloaded model to $model_path ($(du -h "$model_path" | cut -f1) used)"
+                size=$(du -h "$model_path" | cut -f1)
+                echo "✅ Model downloaded successfully ($size): $model_path"
                 return 0
+            else
+                echo "❌ Download failed or file is empty. Retrying..."
+                rm -f "$model_path" 2>/dev/null
+                attempt=$((attempt+1))
+                sleep 2
             fi
-            
-            echo "Download failed or file is empty. Retrying..."
-            rm -f "$model_path"
-            attempt=$((attempt+1))
-            sleep 2
         done
         
-        echo "Failed to download model after $max_attempts attempts"
-        return 1
+        echo "❌ Failed to download model after $max_attempts attempts: $model_url"
+        exit 1
     else
-        echo "Model already exists at $model_path ($(du -h "$model_path" | cut -f1) used)"
-        return 0
+        size=$(du -h "$model_path" | cut -f1)
+        echo "✅ Model already exists ($size): $model_path"
     fi
 }
 
-# Download models
-download_model "/tmp/models/MFD/YOLO/yolo_v8_ft.pt" "https://huggingface.co/marcosremar2/mineru-models/resolve/main/yolo_v8_ft.pt"
-download_model "/tmp/models/MFR/unimernet/unimernet_small.pth" "https://huggingface.co/marcosremar2/mineru-models/resolve/main/unimernet_small.pth"
-download_model "/tmp/models/table/rapid/rapid_table.pt" "https://huggingface.co/marcosremar2/mineru-models/resolve/main/rapid_table.pt"
-download_model "/tmp/models/layout/doclayout/doclayout_yolo.pt" "https://huggingface.co/marcosremar2/mineru-models/resolve/main/doclayout_yolo.pt"
+# Download and verify all required models
+echo "Verifying model files..."
+download_model_with_validation "/tmp/models/layout/doclayout/doclayout_yolo.pt" "https://huggingface.co/marcosremar2/mineru-models/resolve/main/doclayout_yolo.pt"
+download_model_with_validation "/tmp/models/MFD/YOLO/yolo_v8_ft.pt" "https://huggingface.co/marcosremar2/mineru-models/resolve/main/yolo_v8_ft.pt"
 
-# Verify magic-pdf installation and functionality
+# List all model files for verification
+echo "Model files verification:"
+find /tmp/models -type f -exec ls -la {} \;
+
+# Verify magic-pdf exists and is executable
+echo "Checking magic-pdf installation..."
 if command -v magic-pdf &> /dev/null; then
-    echo "Testing magic-pdf command..."
-    magic-pdf --version || echo "magic-pdf command exists but may not be functioning properly"
+    echo "magic-pdf found. Testing version:"
+    magic-pdf --version
 else
-    echo "Warning: magic-pdf command still not available. Will use PyMuPDF as fallback."
+    echo "magic-pdf command not found. Installing:"
+    pip install --no-cache-dir minerupdf==1.3.6
+    if command -v magic-pdf &> /dev/null; then
+        echo "magic-pdf installed successfully. Testing version:"
+        magic-pdf --version
+    else
+        echo "Failed to install magic-pdf. Check PATH and installation."
+        echo "PATH: $PATH"
+        pip list | grep miner
+        exit 1
+    fi
 fi
 
 # Start the Flask application

requirements.txt

@@ -4,5 +4,9 @@ werkzeug==2.3.7
 Pillow>=9.0.0
 numpy>=1.20.0
 requests>=2.25.0
-pymupdf>=1.20.0
-minerupdf 
+opencv-python-headless>=4.5.0
+torch>=1.8.0
+torchvision>=0.9.0
+transformers>=4.15.0
+huggingface_hub>=0.11.0
+minerupdf==1.3.6