Fix tokenizer loading and file permissions

Dockerfile

@@ -2,29 +2,34 @@ FROM python:3.9-slim
 
 WORKDIR /app
 
+# Install system dependencies
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     build-essential \
     && rm -rf /var/lib/apt/lists/*
 
+# Create non-root user
 RUN useradd -m -u 1000 user && \
     chown -R user:user /app
 
+# Set environment variables
 ENV TRANSFORMERS_CACHE=/app/.cache/huggingface
 ENV HF_HOME=/app/.cache/huggingface
 ENV PORT=7860
 
+# Create cache directory with proper permissions
 RUN mkdir -p /app/.cache/huggingface && \
     chown -R user:user /app/.cache
 
-USER user
+# Copy files first
+COPY --chown=user:user . .
 
-COPY --chown=user:user requirements.txt .
+# Switch to non-root user
+USER user
 
+# Install dependencies
 RUN pip install --no-cache-dir --user -r requirements.txt
 
-COPY --chown=user:user . .
-
 EXPOSE 7860
 
 CMD ["python", "app.py"]

app.py

@@ -1,14 +1,29 @@
 from flask import Flask, request, render_template
-from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
+from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline, RobertaTokenizerFast
 from collections import Counter
 import datetime, json
+import os
 
 app = Flask(__name__)
 
-model = AutoModelForTokenClassification.from_pretrained("roberta_model", local_files_only=True)
-tokenizer = AutoTokenizer.from_pretrained("roberta_model", local_files_only=True)
-
-nlp = pipeline("token-classification", model=model, tokenizer=tokenizer, aggregation_strategy="simple")
+try:
+    # Try loading the model first
+    model = AutoModelForTokenClassification.from_pretrained("roberta_model", local_files_only=True)
+    
+    # Load tokenizer using RobertaTokenizerFast directly
+    tokenizer = RobertaTokenizerFast(
+        vocab_file="roberta_model/vocab.json",
+        merges_file="roberta_model/merges.txt",
+        tokenizer_file="roberta_model/tokenizer.json"
+    )
+    
+    # Initialize the pipeline
+    nlp = pipeline("token-classification", model=model, tokenizer=tokenizer, aggregation_strategy="simple")
+    print("Model and tokenizer loaded successfully!")
+    
+except Exception as e:
+    print(f"Error loading model or tokenizer: {str(e)}")
+    raise
 
 @app.route("/", methods=["GET", "POST"])
 def index():
@@ -37,4 +52,5 @@ def index():
                            tag_labels=tag_labels, tag_counts=tag_counts)
 
 if __name__ == "__main__":
-    app.run(debug=True)
+    port = int(os.environ.get("PORT", 7860))
+    app.run(host="0.0.0.0", port=port)