Fix permission issues and improve model downloading

Dockerfile

@@ -26,6 +26,7 @@ RUN apt-get update && apt-get install -y \
     libxrender-dev \
     pkg-config \
     libcairo2-dev \
+    sudo \
     && rm -rf /var/lib/apt/lists/*
 
 # Configure ImageMagick policy to allow PDF conversion (needed for sample PDF creation)
@@ -53,49 +54,58 @@ RUN pip install --no-cache-dir -e ".[full]"
 # Install additional dependencies for the web application
 RUN pip install --no-cache-dir flask==2.3.3 flask-cors==4.0.0 werkzeug==2.3.7
 
-# Create directories for uploads and output
-RUN mkdir -p /tmp/pdf_uploads /tmp/pdf_output
-RUN mkdir -p /tmp/samples
-
-# Create models directory structure
-RUN mkdir -p /tmp/models/MFD/YOLO
-RUN mkdir -p /tmp/models/MFR/unimernet
-RUN mkdir -p /tmp/models/table/rapid
-RUN mkdir -p /tmp/models/layout/doclayout
+# Create a non-root user for Hugging Face Spaces
+RUN useradd -m -u 1000 user && \
+    echo "user ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/user && \
+    chmod 0440 /etc/sudoers.d/user
+
+# Create directories with proper permissions
+RUN mkdir -p /tmp/pdf_uploads /tmp/pdf_output && \
+    chmod -R 777 /tmp/pdf_uploads && \
+    chmod -R 777 /tmp/pdf_output && \
+    mkdir -p /tmp/samples && \
+    chmod -R 777 /tmp/samples
+
+# Create models directory structure with proper permissions
+RUN mkdir -p /tmp/models/MFD/YOLO && \
+    mkdir -p /tmp/models/MFR/unimernet && \
+    mkdir -p /tmp/models/table/rapid && \
+    mkdir -p /tmp/models/layout/doclayout && \
+    chmod -R 777 /tmp/models
 
 # Download model weights
-RUN echo "Downloading MinerU model weights..."
 WORKDIR /tmp/models
 
-# Download the YOLO model for formula detection
-RUN wget -q https://huggingface.co/opendatalab/MinerU/resolve/main/models/mfd/yolo_v8_mfd.pt -O /tmp/models/MFD/YOLO/yolo_v8_ft.pt || echo "Failed to download yolo_v8_ft.pt"
-RUN wget -q https://huggingface.co/opendatalab/MinerU/resolve/main/models/mfd/yolo_v8_mfd.pt -O /tmp/models/MFD/YOLO/yolo_v8_mfd.pt || echo "Failed to download yolo_v8_mfd.pt"
-
-# Download the UniMERNet model for formula recognition
-RUN wget -q https://huggingface.co/opendatalab/MinerU/resolve/main/models/mfr/unimernet_small.pth -O /tmp/models/MFR/unimernet/unimernet_small.pth || echo "Failed to download unimernet_small.pth"
-
-# Download the Rapid Table detection model
-RUN wget -q https://huggingface.co/opendatalab/MinerU/resolve/main/models/table/rapid_table.pt -O /tmp/models/table/rapid/rapid_table.pt || echo "Failed to download rapid_table.pt"
-RUN wget -q https://huggingface.co/opendatalab/MinerU/resolve/main/models/table/slanet_plus.pt -O /tmp/models/table/rapid/slanet_plus.pt || echo "Failed to download slanet_plus.pt"
-
-# Download the DocLayout YOLO model
-RUN wget -q https://huggingface.co/opendatalab/MinerU/resolve/main/models/layout/doclayout_yolo.pt -O /tmp/models/layout/doclayout/doclayout_yolo.pt || echo "Failed to download doclayout_yolo.pt"
-
-# Create a non-root user for Hugging Face Spaces
-# This is critical for permissions on HF Spaces
-RUN useradd -m -u 1000 user
+# Download the YOLO model for formula detection (using curl with progress and retry)
+RUN apt-get update && apt-get install -y curl && \
+    curl -L --retry 5 --retry-delay 5 -o /tmp/models/MFD/YOLO/yolo_v8_ft.pt https://huggingface.co/opendatalab/MinerU/resolve/main/models/mfd/yolo_v8_mfd.pt && \
+    curl -L --retry 5 --retry-delay 5 -o /tmp/models/MFD/YOLO/yolo_v8_mfd.pt https://huggingface.co/opendatalab/MinerU/resolve/main/models/mfd/yolo_v8_mfd.pt && \
+    curl -L --retry 5 --retry-delay 5 -o /tmp/models/MFR/unimernet/unimernet_small.pth https://huggingface.co/opendatalab/MinerU/resolve/main/models/mfr/unimernet_small.pth && \
+    curl -L --retry 5 --retry-delay 5 -o /tmp/models/table/rapid/rapid_table.pt https://huggingface.co/opendatalab/MinerU/resolve/main/models/table/rapid_table.pt && \
+    curl -L --retry 5 --retry-delay 5 -o /tmp/models/table/rapid/slanet_plus.pt https://huggingface.co/opendatalab/MinerU/resolve/main/models/table/slanet_plus.pt && \
+    curl -L --retry 5 --retry-delay 5 -o /tmp/models/layout/doclayout/doclayout_yolo.pt https://huggingface.co/opendatalab/MinerU/resolve/main/models/layout/doclayout_yolo.pt && \
+    ls -la /tmp/models/MFD/YOLO/ && \
+    ls -la /tmp/models/MFR/unimernet/ && \
+    ls -la /tmp/models/table/rapid/ && \
+    ls -la /tmp/models/layout/doclayout/
+
+# Verify file sizes (files should not be empty)
+RUN find /tmp/models -type f -size 0 -exec echo "Warning: Empty file {}" \;
+
+# Create app directory
 RUN mkdir -p /app/samples && chown -R user:user /app
 
-# Set permissions for model files
-RUN chown -R user:user /tmp/models
-
 # Copy the application files
 WORKDIR /app
 COPY . /app/
 
 # Fix permissions for the user
-RUN chown -R user:user /app
-RUN mkdir -p /home/user/.config/magic_pdf && chown -R user:user /home/user/.config
+RUN chown -R user:user /app && \
+    mkdir -p /home/user/.config/magic_pdf && \
+    chown -R user:user /home/user/.config && \
+    chown -R user:user /tmp/pdf_uploads && \
+    chown -R user:user /tmp/pdf_output && \
+    chown -R user:user /tmp/models
 
 # Expose the port
 EXPOSE 7860

app.py

@@ -9,12 +9,15 @@ import tempfile
 import json
 import time
 import shutil
+import sys
 
 app = Flask(__name__)
 CORS(app)
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger()
+logger.addHandler(logging.StreamHandler(sys.stdout))
 
 # Constants
 UPLOAD_FOLDER = os.environ.get('UPLOAD_FOLDER', '/tmp/pdf_uploads')
@@ -22,9 +25,34 @@ OUTPUT_FOLDER = os.environ.get('OUTPUT_FOLDER', '/tmp/pdf_output')
 ALLOWED_EXTENSIONS = {'pdf'}
 MAX_FILE_SIZE = 50 * 1024 * 1024  # 50MB
 
-# Ensure the directories exist
-os.makedirs(UPLOAD_FOLDER, exist_ok=True)
-os.makedirs(OUTPUT_FOLDER, exist_ok=True)
+# Ensure the directories exist and are writable
+try:
+    os.makedirs(UPLOAD_FOLDER, exist_ok=True)
+    os.makedirs(OUTPUT_FOLDER, exist_ok=True)
+    # Test if we can write to these directories
+    test_upload_file = os.path.join(UPLOAD_FOLDER, '.test_file')
+    test_output_file = os.path.join(OUTPUT_FOLDER, '.test_file')
+    
+    with open(test_upload_file, 'w') as f:
+        f.write('test')
+    os.remove(test_upload_file)
+    
+    with open(test_output_file, 'w') as f:
+        f.write('test')
+    os.remove(test_output_file)
+    
+    logging.info(f"Using upload folder: {UPLOAD_FOLDER}")
+    logging.info(f"Using output folder: {OUTPUT_FOLDER}")
+except Exception as e:
+    # Fall back to user's home directory
+    logging.warning(f"Cannot use specified directories: {str(e)}")
+    USER_HOME = os.path.expanduser("~")
+    UPLOAD_FOLDER = os.path.join(USER_HOME, 'pdf_uploads')
+    OUTPUT_FOLDER = os.path.join(USER_HOME, 'pdf_output')
+    os.makedirs(UPLOAD_FOLDER, exist_ok=True)
+    os.makedirs(OUTPUT_FOLDER, exist_ok=True)
+    logging.info(f"Using fallback upload folder: {UPLOAD_FOLDER}")
+    logging.info(f"Using fallback output folder: {OUTPUT_FOLDER}")
 
 # Function to check if file extension is allowed
 def allowed_file(filename):
@@ -112,6 +140,13 @@ HTML_TEMPLATE = """
         a:hover {
             text-decoration: underline;
         }
+        .system-info {
+            font-size: 12px;
+            color: #777;
+            margin-top: 20px;
+            padding-top: 10px;
+            border-top: 1px solid #ddd;
+        }
     </style>
 </head>
 <body>
@@ -156,6 +191,12 @@ HTML_TEMPLATE = """
                 curl -X POST -F "file=@your_file.pdf" https://marcosremar2-mineruapi.hf.space/api/convert
             </div>
         </div>
+        
+        <div class="system-info">
+            <p>Upload directory: ${UPLOAD_FOLDER}</p>
+            <p>Output directory: ${OUTPUT_FOLDER}</p>
+            <p>Max file size: ${MAX_FILE_SIZE/(1024*1024)}MB</p>
+        </div>
     </div>
 
     <script>
@@ -237,7 +278,10 @@ HTML_TEMPLATE = """
 # Route for the main page
 @app.route('/')
 def index():
-    return render_template_string(HTML_TEMPLATE, MAX_FILE_SIZE=MAX_FILE_SIZE)
+    return render_template_string(HTML_TEMPLATE, 
+                                 MAX_FILE_SIZE=MAX_FILE_SIZE,
+                                 UPLOAD_FOLDER=UPLOAD_FOLDER,
+                                 OUTPUT_FOLDER=OUTPUT_FOLDER)
 
 # Route for the health check
 @app.route('/health')
@@ -263,11 +307,26 @@ def health_check():
             "doclayout_model": os.path.exists("/tmp/models/layout/doclayout/doclayout_yolo.pt")
         }
         
+        # Check directory permissions
+        dir_permissions = {
+            "upload_dir": {
+                "path": UPLOAD_FOLDER,
+                "exists": os.path.exists(UPLOAD_FOLDER),
+                "writable": os.access(UPLOAD_FOLDER, os.W_OK)
+            },
+            "output_dir": {
+                "path": OUTPUT_FOLDER,
+                "exists": os.path.exists(OUTPUT_FOLDER),
+                "writable": os.access(OUTPUT_FOLDER, os.W_OK)
+            }
+        }
+        
         return jsonify({
             "status": "healthy",
             "version": version,
             "gpu_available": gpu_available,
             "model_dirs": model_dirs,
+            "directories": dir_permissions,
             "timestamp": time.time()
         })
     except Exception as e:
@@ -314,12 +373,26 @@ def convert_pdf():
         # Create a unique session ID
         session_id = str(uuid.uuid4())
         session_dir = os.path.join(OUTPUT_FOLDER, session_id)
-        os.makedirs(session_dir, exist_ok=True)
+        
+        try:
+            os.makedirs(session_dir, exist_ok=True)
+            logging.info(f"Created session directory: {session_dir}")
+        except PermissionError as e:
+            logging.error(f"Permission error creating directory {session_dir}: {str(e)}")
+            # Try using a temp directory instead
+            session_dir = tempfile.mkdtemp(prefix="minerupdf_")
+            logging.info(f"Using temporary directory instead: {session_dir}")
         
         # Save the uploaded file
         input_path = os.path.join(UPLOAD_FOLDER, f"{session_id}_{file.filename}")
-        file.save(input_path)
-        logging.info(f"Saved uploaded file to {input_path}")
+        try:
+            file.save(input_path)
+            logging.info(f"Saved uploaded file to {input_path}")
+        except PermissionError:
+            # Try saving in the temp directory
+            input_path = os.path.join(session_dir, f"{session_id}_{file.filename}")
+            file.save(input_path)
+            logging.info(f"Saved uploaded file to alternate location: {input_path}")
         
         # Get output file paths
         base_filename = os.path.splitext(os.path.basename(file.filename))[0]
@@ -343,6 +416,7 @@ def convert_pdf():
                 "--mfd"
             ]
             
+            logging.info(f"Running command: {' '.join(cmd)}")
             process = subprocess.run(cmd, capture_output=True, text=True, check=False)
             
             if process.returncode != 0:
@@ -391,7 +465,7 @@ def convert_pdf():
     finally:
         # Clean up the input file
         try:
-            if os.path.exists(input_path):
+            if 'input_path' in locals() and os.path.exists(input_path):
                 os.remove(input_path)
         except Exception as e:
             logging.warning(f"Failed to clean up input file: {str(e)}")

entrypoint.sh

@@ -14,6 +14,33 @@ nvidia-smi || echo "No NVIDIA GPU detected, running in CPU mode"
 echo "MinerU version:"
 magic-pdf --version || echo "Error: MinerU magic-pdf not found"
 
+# Ensure output directories have proper permissions
+echo "Checking output directories..."
+for DIR in "/tmp/pdf_uploads" "/tmp/pdf_output" "/tmp/models"; do
+    if [ -d "$DIR" ]; then
+        echo "Ensuring $DIR is writable..."
+        chmod -R 777 "$DIR" || echo "Could not set permissions on $DIR"
+    else
+        echo "Creating directory $DIR..."
+        mkdir -p "$DIR" || echo "Could not create $DIR"
+        chmod -R 777 "$DIR" || echo "Could not set permissions on $DIR"
+    fi
+done
+
+# Fallback to user's home directory if tmp is not writable
+if ! touch "/tmp/test_write_permission" 2>/dev/null; then
+    echo "Warning: /tmp directory is not writable, using $HOME instead"
+    export UPLOAD_FOLDER="$HOME/pdf_uploads"
+    export OUTPUT_FOLDER="$HOME/pdf_output"
+    mkdir -p "$UPLOAD_FOLDER" "$OUTPUT_FOLDER"
+    echo "Using alternative upload folder: $UPLOAD_FOLDER"
+    echo "Using alternative output folder: $OUTPUT_FOLDER"
+else
+    rm -f "/tmp/test_write_permission"
+    export UPLOAD_FOLDER="/tmp/pdf_uploads"
+    export OUTPUT_FOLDER="/tmp/pdf_output"
+fi
+
 # Create directories for models if they don't exist
 mkdir -p /tmp/models/MFD/YOLO
 mkdir -p /tmp/models/MFR/unimernet
@@ -42,11 +69,22 @@ MODEL_URLS=(
 )
 
 for i in "${!MODEL_FILES[@]}"; do
-    if [ ! -f "${MODEL_FILES[$i]}" ]; then
+    if [ ! -f "${MODEL_FILES[$i]}" ] || [ ! -s "${MODEL_FILES[$i]}" ]; then
         echo "Downloading ${MODEL_FILES[$i]}..."
-        wget -q "${MODEL_URLS[$i]}" -O "${MODEL_FILES[$i]}" || echo "Failed to download ${MODEL_FILES[$i]}"
+        curl -L --retry 5 --retry-delay 5 -o "${MODEL_FILES[$i]}" "${MODEL_URLS[$i]}" || echo "Failed to download ${MODEL_FILES[$i]}"
+        
+        # Verify file size
+        if [ -f "${MODEL_FILES[$i]}" ]; then
+            SIZE=$(stat -c%s "${MODEL_FILES[$i]}" 2>/dev/null || stat -f%z "${MODEL_FILES[$i]}")
+            if [ "$SIZE" -eq 0 ]; then
+                echo "Warning: Downloaded file ${MODEL_FILES[$i]} is empty!"
+            else
+                echo "${MODEL_FILES[$i]} downloaded successfully (${SIZE} bytes)"
+            fi
+        fi
     else
-        echo "${MODEL_FILES[$i]} already exists."
+        SIZE=$(stat -c%s "${MODEL_FILES[$i]}" 2>/dev/null || stat -f%z "${MODEL_FILES[$i]}")
+        echo "${MODEL_FILES[$i]} already exists (${SIZE} bytes)."
     fi
 done
 
@@ -58,11 +96,11 @@ mkdir -p $HOME/.config/magic_pdf
 echo "Downloading sample PDF for testing..."
 if [ ! -f "$HOME/samples/sample.pdf" ]; then
     # Download a simple paper from arXiv (using a small one for quick processing)
-    wget -q "https://arxiv.org/pdf/2201.08239.pdf" -O "$HOME/samples/sample.pdf" || true
+    curl -L --retry 3 --retry-delay 3 -o "$HOME/samples/sample.pdf" "https://arxiv.org/pdf/2201.08239.pdf" || true
     
     # If that fails, try another source
     if [ ! -s "$HOME/samples/sample.pdf" ]; then
-        wget -q "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf" -O "$HOME/samples/sample.pdf" || true
+        curl -L --retry 3 -o "$HOME/samples/sample.pdf" "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf" || true
     fi
     
     # If both fail, create a simple PDF with text
@@ -122,6 +160,10 @@ ls -la /tmp/models/MFR/unimernet/ || echo "UniMERNet models directory issue"
 ls -la /tmp/models/table/rapid/ || echo "Table models directory issue"
 ls -la /tmp/models/layout/doclayout/ || echo "Layout models directory issue"
 
+# Check file sizes to ensure they're not empty
+echo "Checking model file sizes:"
+find /tmp/models -type f -size 0c -exec echo "Warning: Empty file {}" \;
+
 # Start the Flask application
 echo "Starting Flask application..."
 python /app/app.py