Fix entrypoint script to properly install and verify magic-pdf tool

entrypoint.sh

@@ -1,169 +1,111 @@
 #!/bin/bash
 set -e
 
-# Activate virtual environment
-source /opt/mineru_venv/bin/activate
-
-# Display GPU information
-echo "===== Application Startup at $(date +'%Y-%m-%d %H:%M:%S') ====="
-echo ""
-echo "Checking NVIDIA GPU status:"
-nvidia-smi || echo "No NVIDIA GPU detected, running in CPU mode"
-
-# Display MinerU version
-echo "MinerU version:"
-magic-pdf --version || echo "Error: MinerU magic-pdf not found"
-
-# Ensure output directories have proper permissions
-echo "Checking output directories..."
-for DIR in "/tmp/pdf_uploads" "/tmp/pdf_output" "/tmp/models"; do
-    if [ -d "$DIR" ]; then
-        echo "Ensuring $DIR is writable..."
-        chmod -R 777 "$DIR" || echo "Could not set permissions on $DIR"
-    else
-        echo "Creating directory $DIR..."
-        mkdir -p "$DIR" || echo "Could not create $DIR"
-        chmod -R 777 "$DIR" || echo "Could not set permissions on $DIR"
-    fi
-done
-
-# Fallback to user's home directory if tmp is not writable
-if ! touch "/tmp/test_write_permission" 2>/dev/null; then
-    echo "Warning: /tmp directory is not writable, using $HOME instead"
-    export UPLOAD_FOLDER="$HOME/pdf_uploads"
-    export OUTPUT_FOLDER="$HOME/pdf_output"
-    mkdir -p "$UPLOAD_FOLDER" "$OUTPUT_FOLDER"
-    echo "Using alternative upload folder: $UPLOAD_FOLDER"
-    echo "Using alternative output folder: $OUTPUT_FOLDER"
-else
-    rm -f "/tmp/test_write_permission"
-    export UPLOAD_FOLDER="/tmp/pdf_uploads"
-    export OUTPUT_FOLDER="/tmp/pdf_output"
-fi
-
-# Create directories for models if they don't exist
+# Show environment information
+echo "Starting entrypoint.sh..."
+echo "User: $(whoami)"
+echo "Current directory: $(pwd)"
+echo "Directory listing:"
+ls -la
+
+# Create necessary directories
+mkdir -p /tmp/pdf_uploads
+mkdir -p /tmp/pdf_output
 mkdir -p /tmp/models/MFD/YOLO
 mkdir -p /tmp/models/MFR/unimernet
 mkdir -p /tmp/models/table/rapid
 mkdir -p /tmp/models/layout/doclayout
 
-# Check if model files exist, if not, download them
-echo "Checking model files..."
-MODEL_FILES=(
-    "/tmp/models/MFD/YOLO/yolo_v8_ft.pt"
-    "/tmp/models/MFD/YOLO/yolo_v8_mfd.pt"
-    "/tmp/models/MFR/unimernet/unimernet_small.pth"
-    "/tmp/models/table/rapid/rapid_table.pt"
-    "/tmp/models/table/rapid/slanet_plus.pt"
-    "/tmp/models/layout/doclayout/doclayout_yolo.pt"
-)
-
-MODELS_REPO="https://huggingface.co/opendatalab/MinerU/resolve/main/models"
-MODEL_URLS=(
-    "${MODELS_REPO}/mfd/yolo_v8_mfd.pt"
-    "${MODELS_REPO}/mfd/yolo_v8_mfd.pt"
-    "${MODELS_REPO}/mfr/unimernet_small.pth"
-    "${MODELS_REPO}/table/rapid_table.pt"
-    "${MODELS_REPO}/table/slanet_plus.pt"
-    "${MODELS_REPO}/layout/doclayout_yolo.pt"
-)
+# Set permissions (ensure directories are writable)
+chmod -R 777 /tmp/pdf_uploads
+chmod -R 777 /tmp/pdf_output
+chmod -R 777 /tmp/models
 
-for i in "${!MODEL_FILES[@]}"; do
-    if [ ! -f "${MODEL_FILES[$i]}" ] || [ ! -s "${MODEL_FILES[$i]}" ]; then
-        echo "Downloading ${MODEL_FILES[$i]}..."
-        curl -L --retry 5 --retry-delay 5 -o "${MODEL_FILES[$i]}" "${MODEL_URLS[$i]}" || echo "Failed to download ${MODEL_FILES[$i]}"
-        
-        # Verify file size
-        if [ -f "${MODEL_FILES[$i]}" ]; then
-            SIZE=$(stat -c%s "${MODEL_FILES[$i]}" 2>/dev/null || stat -f%z "${MODEL_FILES[$i]}")
-            if [ "$SIZE" -eq 0 ]; then
-                echo "Warning: Downloaded file ${MODEL_FILES[$i]} is empty!"
-            else
-                echo "${MODEL_FILES[$i]} downloaded successfully (${SIZE} bytes)"
-            fi
-        fi
+# Check if magic-pdf command exists
+if ! command -v magic-pdf &> /dev/null; then
+    echo "Warning: magic-pdf command not found. Installing MinerU PDF processing tools..."
+    pip install --no-cache-dir minerupdf
+    
+    # Verify installation
+    if command -v magic-pdf &> /dev/null; then
+        echo "Successfully installed magic-pdf."
     else
-        SIZE=$(stat -c%s "${MODEL_FILES[$i]}" 2>/dev/null || stat -f%z "${MODEL_FILES[$i]}")
-        echo "${MODEL_FILES[$i]} already exists (${SIZE} bytes)."
+        echo "Failed to install magic-pdf. Will use PyMuPDF fallback."
+        pip install --no-cache-dir pymupdf
     fi
-done
-
-# Create a samples directory
-mkdir -p $HOME/samples
-mkdir -p $HOME/.config/magic_pdf
+else
+    echo "magic-pdf command is already installed."
+fi
 
-# Download a sample PDF for testing if it doesn't exist
-echo "Downloading sample PDF for testing..."
-if [ ! -f "$HOME/samples/sample.pdf" ]; then
-    # Download a simple paper from arXiv (using a small one for quick processing)
-    curl -L --retry 3 --retry-delay 3 -o "$HOME/samples/sample.pdf" "https://arxiv.org/pdf/2201.08239.pdf" || true
-    
-    # If that fails, try another source
-    if [ ! -s "$HOME/samples/sample.pdf" ]; then
-        curl -L --retry 3 -o "$HOME/samples/sample.pdf" "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf" || true
-    fi
+# Configure magic-pdf
+mkdir -p ~/.config/magic_pdf
+cat > ~/.config/magic_pdf/magic-pdf.json << EOL
+{
+  "device-mode": "cpu",
+  "layout-config": {
+    "model": "doclayout_yolo",
+    "enable": true
+  },
+  "formula-config": {
+    "mfd_model": "yolo_v8_mfd",
+    "mfr_model": "unimernet_small",
+    "enable": true
+  },
+  "table-config": {
+    "model": "rapid_table",
+    "sub_model": "slanet_plus",
+    "enable": true,
+    "max_time": 400
+  }
+}
+EOL
+
+# Download required model files if they don't exist or are empty
+function download_model() {
+    local model_path=$1
+    local model_url=$2
+    local max_attempts=3
+    local attempt=1
     
-    # If both fail, create a simple PDF with text
-    if [ ! -s "$HOME/samples/sample.pdf" ]; then
-        echo "Failed to download sample PDF, creating a simple PDF text file..."
-        echo "This is a sample PDF document for testing MinerU.
-        
-        MinerU is a high-quality tool for converting PDF to Markdown and JSON formats.
-        
-        This file was created for testing purposes." > "$HOME/samples/sample.txt"
+    if [ ! -f "$model_path" ] || [ ! -s "$model_path" ]; then
+        echo "Downloading model to $model_path from $model_url"
+        while [ $attempt -le $max_attempts ]; do
+            echo "Download attempt $attempt of $max_attempts..."
+            curl -L "$model_url" --output "$model_path" --retry 3 --retry-delay 2
+            
+            if [ -f "$model_path" ] && [ -s "$model_path" ]; then
+                echo "Successfully downloaded model to $model_path ($(du -h "$model_path" | cut -f1) used)"
+                return 0
+            fi
+            
+            echo "Download failed or file is empty. Retrying..."
+            rm -f "$model_path"
+            attempt=$((attempt+1))
+            sleep 2
+        done
         
-        # Try using different methods to create a PDF
-        if command -v convert &> /dev/null; then
-            convert -size 612x792 -background white -fill black caption:@"$HOME/samples/sample.txt" "$HOME/samples/sample.pdf"
-        else
-            echo "WARNING: Could not create a sample PDF file automatically."
-        fi
+        echo "Failed to download model after $max_attempts attempts"
+        return 1
+    else
+        echo "Model already exists at $model_path ($(du -h "$model_path" | cut -f1) used)"
+        return 0
     fi
-fi
-
-# Create the magic-pdf.json config file with paths to the downloaded models
-echo "Creating magic-pdf.json configuration file..."
-cat > "$HOME/.config/magic_pdf/magic-pdf.json" << EOF
-{
-    "device-mode": "gpu",
-    "models-path": "/tmp/models",
-    "layout-config": {
-        "model": "doclayout_yolo",
-        "model_path": "/tmp/models/layout/doclayout/doclayout_yolo.pt",
-        "enable": true
-    },
-    "formula-config": {
-        "mfd_model": "yolo_v8_mfd",
-        "mfd_model_path": "/tmp/models/MFD/YOLO/yolo_v8_mfd.pt",
-        "mfr_model": "unimernet_small",
-        "mfr_model_path": "/tmp/models/MFR/unimernet/unimernet_small.pth",
-        "enable": true
-    },
-    "table-config": {
-        "model": "rapid_table",
-        "model_path": "/tmp/models/table/rapid/rapid_table.pt",
-        "sub_model": "slanet_plus",
-        "sub_model_path": "/tmp/models/table/rapid/slanet_plus.pt",
-        "enable": true,
-        "max_time": 400
-    }
 }
-EOF
 
-# Also create it in the home directory as some versions of MinerU look for it there
-cp "$HOME/.config/magic_pdf/magic-pdf.json" "$HOME/magic-pdf.json"
+# Download models
+download_model "/tmp/models/MFD/YOLO/yolo_v8_ft.pt" "https://huggingface.co/marcosremar2/mineru-models/resolve/main/yolo_v8_ft.pt"
+download_model "/tmp/models/MFR/unimernet/unimernet_small.pth" "https://huggingface.co/marcosremar2/mineru-models/resolve/main/unimernet_small.pth"
+download_model "/tmp/models/table/rapid/rapid_table.pt" "https://huggingface.co/marcosremar2/mineru-models/resolve/main/rapid_table.pt"
+download_model "/tmp/models/layout/doclayout/doclayout_yolo.pt" "https://huggingface.co/marcosremar2/mineru-models/resolve/main/doclayout_yolo.pt"
 
-# List model files to verify they're present
-echo "Verifying model files:"
-ls -la /tmp/models/MFD/YOLO/ || echo "YOLO models directory issue"
-ls -la /tmp/models/MFR/unimernet/ || echo "UniMERNet models directory issue"
-ls -la /tmp/models/table/rapid/ || echo "Table models directory issue"
-ls -la /tmp/models/layout/doclayout/ || echo "Layout models directory issue"
-
-# Check file sizes to ensure they're not empty
-echo "Checking model file sizes:"
-find /tmp/models -type f -size 0c -exec echo "Warning: Empty file {}" \;
+# Verify magic-pdf installation and functionality
+if command -v magic-pdf &> /dev/null; then
+    echo "Testing magic-pdf command..."
+    magic-pdf --version || echo "magic-pdf command exists but may not be functioning properly"
+else
+    echo "Warning: magic-pdf command still not available. Will use PyMuPDF as fallback."
+fi
 
 # Start the Flask application
 echo "Starting Flask application..."
-python /app/app.py 
+python app.py