Update app.py

app.py

@@ -22,7 +22,7 @@ bnb_config = BitsAndBytesConfig(
     load_in_4bit=True,
     bnb_4bit_quant_type="nf4",
     bnb_4bit_use_double_quant=True,
-    bnb_4bit_compute_dtype=torch.float16,  # T4 uses FP16 Tensor Cores
+    bnb_4bit_compute_dtype=torch.float16,
 )
 
 print("⏳ Loading Base Model in 4-bit onto T4 GPU...")
@@ -36,7 +36,15 @@ base_model = AutoModelForCausalLM.from_pretrained(
 print("⏳ Attaching Fine-Tuned LoRA Adapter...")
 model = PeftModel.from_pretrained(base_model, ADAPTER_NAME)
 model.eval()
-print("✅ Model & Adapter loaded on Nvidia T4!")
+
+print("⚡ Compiling model with torch.compile() for accelerated token generation...")
+# Optimizes the execution graph for faster inference throughput on T4
+try:
+    model = torch.compile(model, mode="reduce-overhead", fullgraph=False)
+except Exception as e:
+    print(f"⚠️ torch.compile() skipped or failed: {e}. Falling back to standard execution.")
+
+print("✅ Model, Adapter & Compilation ready on Nvidia T4!")
 
 SYSTEM_PROMPT = (
     "You are an expert human academic writer. Rewrite the provided AI text into natural, human-written prose "
@@ -57,7 +65,6 @@ def humanize_full_essay(essay_text, temperature):
     rewritten_paragraphs = []
 
     for p in paragraphs:
-        # Preserve short headings verbatim
         if len(p.split()) <= 8 and not p.endswith("."):
             rewritten_paragraphs.append(p)
             yield "\n\n".join(rewritten_paragraphs)
@@ -104,7 +111,6 @@ def humanize_full_essay(essay_text, temperature):
 
         rewritten_paragraphs.append(partial.strip())
         
-        # Clear cached VRAM between paragraphs to keep T4 memory usage flat
         if torch.cuda.is_available():
             torch.cuda.empty_cache()
 
@@ -114,7 +120,7 @@ demo = gr.Interface(
         gr.Textbox(lines=16, label="Paste Full AI Essay"),
         gr.Slider(minimum=0.3, maximum=1.0, value=0.75, step=0.05, label="Temperature"),
     ],
-    outputs=gr.Textbox(lines=16, label="Humanized Output (Nvidia T4 4-Bit)"),
+    outputs=gr.Textbox(lines=16, label="Humanized Output (Nvidia T4 - Compiled)"),
     title="Academic Humanizer (Qwen2.5-14B Fine-Tuned)",
 )