Upload 5 files

app.py

@@ -4,6 +4,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
 import torch
 import sys
 import os
+import gc  # For garbage collection optimization
 
 app = Flask(__name__)
 CORS(app)
@@ -21,13 +22,18 @@ if tokenizer.pad_token is None:
 
 print("✅ Tokenizer loaded!")
 
+# Optimized quantization for 2 vCPU + 18GB RAM
 quantization_config = BitsAndBytesConfig(
     load_in_4bit=True,
     bnb_4bit_compute_dtype=torch.float16,
     bnb_4bit_quant_type="nf4",
-    bnb_4bit_use_double_quant=True,
+    bnb_4bit_use_double_quant=False,  # Disabled for CPU efficiency
 )
 
+# Set CPU threads BEFORE loading model to reduce startup CPU spike
+torch.set_num_threads(2)
+torch.set_num_interop_threads(1)
+
 model = AutoModelForCausalLM.from_pretrained(
     MODEL_NAME,
     quantization_config=quantization_config,
@@ -37,8 +43,14 @@ model = AutoModelForCausalLM.from_pretrained(
     torch_dtype=torch.float16,
 )
 
-print("✅ Model loaded!")
+# Set model to eval mode and optimize for inference
+model.eval()
+for param in model.parameters():
+    param.requires_grad = False
+
+print("✅ Model loaded and optimized!")
 print(f"Device: {model.device}")
+print(f"Threads: {torch.get_num_threads()}")
 print("="*80 + "\n")
 
 HTML_TEMPLATE = """
@@ -146,7 +158,7 @@ HTML_TEMPLATE = """
         <button onclick="generate()" id="generateBtn">💬 Send</button>
         
         <div class="loading" id="loading">
-            <p>⏳ Generating response... Please wait (this may take 30-60 seconds on CPU)</p>
+            <p>⏳ Generating response... Please wait (typically 15-30 seconds on 2 vCPU)</p>
         </div>
         
         <div class="output" id="output"></div>
@@ -237,16 +249,18 @@ def generate():
         sys.stdout.flush()
         
         with torch.no_grad():
-            torch.set_num_threads(2)
             outputs = model.generate(
                 **inputs,
-                max_new_tokens=200,
+                max_new_tokens=150,  # Reduced for faster response
                 do_sample=True,
-                temperature=0.7,
-                top_p=0.9,
+                temperature=0.3,  # Lower temp = faster, more focused
+                top_p=0.85,  # Slightly lower for efficiency
+                top_k=40,  # Limit sampling space
+                repetition_penalty=1.1,  # Prevent loops
                 pad_token_id=tokenizer.pad_token_id,
                 eos_token_id=tokenizer.eos_token_id,
-                use_cache=False
+                use_cache=True,  # Enable KV cache for speed
+                num_beams=1,  # Greedy = faster
             )
         
         full_output = tokenizer.decode(outputs[0], skip_special_tokens=True)
@@ -273,6 +287,13 @@ def generate():
         return jsonify({'error': str(e)}), 500
 
 if __name__ == '__main__':
+    # Force garbage collection after model load
+    import gc
+    gc.collect()
+    
     port = int(os.environ.get('PORT', 7860))
     print(f"🌐 Starting server on port {port}...\n")
-    app.run(host='0.0.0.0', port=port, debug=False)
+    print("💡 CPU usage should normalize after initial model load\n")
+    
+    # Use threaded mode for better concurrency on 2 vCPU
+    app.run(host='0.0.0.0', port=port, debug=False, threaded=True, use_reloader=False)