Fix permissions error: proper cache directory and HF token auth for Llama 3.1

Dockerfile

@@ -34,9 +34,11 @@ RUN pip install -r requirements.txt
 COPY app.py .
 COPY README.md .
 
-# Create HF cache directory
-RUN mkdir -p /.cache
-ENV HF_HOME=/.cache
+# Create HF cache directory with proper permissions
+RUN mkdir -p /app/.cache && chmod -R 777 /app/.cache
+ENV HF_HOME=/app/.cache
+ENV TRANSFORMERS_CACHE=/app/.cache
+ENV HF_DATASETS_CACHE=/app/.cache
 
 # Expose port
 EXPOSE 7860

app.py

@@ -62,13 +62,17 @@ async def load_model():
             
             logger.info("Loading model with transformers...")
             
-            # Use the base model instead of GGUF for better compatibility
+            # Use Llama 3.1 8B Instruct (requires HF token with access)
             base_model_name = "meta-llama/Llama-3.1-8B-Instruct"
             
+            # Get HF token from environment
+            hf_token = os.getenv("HF_TOKEN")
+            
             tokenizer = AutoTokenizer.from_pretrained(
                 base_model_name,
                 use_fast=True,
-                trust_remote_code=True
+                trust_remote_code=True,
+                token=hf_token
             )
             
             model = AutoModelForCausalLM.from_pretrained(
@@ -76,7 +80,8 @@ async def load_model():
                 torch_dtype=torch.float16 if device == "cuda" else torch.float32,
                 device_map="auto" if device == "cuda" else None,
                 trust_remote_code=True,
-                low_cpu_mem_usage=True
+                low_cpu_mem_usage=True,
+                token=hf_token
             )
             
             if device == "cuda":