Update app.py

app.py

@@ -14,8 +14,6 @@ from langchain.schema import LLMResult
 from langchain.llms.base import LLM
 import torch
 import os
-import random
-import PyPDF2
 
 # --------------------------
 # Load Quantized LLaMA 2 via AutoGPTQ
@@ -43,17 +41,28 @@ model = AutoGPTQForCausalLM.from_quantized(
     token=hf_token
 )
 
+model.to(device)
+model.eval()  # Ensure evaluation mode
+
+# --------------------------
 # Manual text generation function
+# --------------------------
 def generate_text(prompt):
     inputs = tokenizer(prompt, return_tensors="pt").to(device)
-    generation_output = model.generate(
-        input_ids=inputs.input_ids,
-        attention_mask=inputs.attention_mask,
-        max_new_tokens=256,
-        do_sample=True,
-        temperature=0.7,
-        top_p=0.9,
-    )
+
+    # Ensure input types are compatible
+    inputs.input_ids = inputs.input_ids.to(torch.int64)
+    inputs.attention_mask = inputs.attention_mask.to(torch.int64)
+
+    with torch.no_grad():
+        generation_output = model.generate(
+            input_ids=inputs.input_ids,
+            attention_mask=inputs.attention_mask,
+            max_new_tokens=256,
+            do_sample=True,
+            temperature=0.7,
+            top_p=0.9,
+        )
     decoded = tokenizer.decode(generation_output[0], skip_special_tokens=True)
     return decoded
 
@@ -95,7 +104,8 @@ whisper_model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-
 
 def transcribe_audio(audio):
     audio_input = whisper_processor(audio["array"], sampling_rate=audio["sampling_rate"], return_tensors="pt")
-    result = whisper_model.generate(**audio_input)
+    with torch.no_grad():
+        result = whisper_model.generate(**audio_input)
     return whisper_processor.batch_decode(result, skip_special_tokens=True)[0]
 
 # --------------------------
@@ -104,8 +114,7 @@ def transcribe_audio(audio):
 def handle_user_input(text):
     prompt = f"<s>[INST] {text} [/INST]"
     response = generate_text(prompt)
-    response = response.split("[/INST]")[-1].strip()
-    return response
+    return response.split("[/INST]")[-1].strip()
 
 def handle_pdf_query(text):
     result = qa_chain(text)
@@ -126,38 +135,13 @@ def upload_and_index_pdf(pdf_file):
     return f"Uploaded and indexed: {os.path.basename(pdf_file.name)}"
 
 # --------------------------
-# Generate Sample Questions from PDF
+# Placeholder for test question generation
 # --------------------------
 def generate_test_questions_from_pdf(pdf_file):
+    # This is a placeholder — implement your own question generation logic
     if pdf_file is None:
-        return "No file uploaded."
-
-    # Read text from PDF
-    pdf_reader = PyPDF2.PdfReader(pdf_file.name)
-    text = ""
-    for page in pdf_reader.pages:
-        text += page.extract_text() + " "
-
-    # Split text into sentences
-    sentences = [s.strip() for s in text.split(".") if s.strip()]
-    if not sentences:
-        return "No extractable text found in PDF."
-
-    # Generate sample questions (randomly)
-    questions = []
-    templates = [
-        "Explain the following: {}?",
-        "What is the main point of: {}?",
-        "Summarize: {}",
-        "Why is {} important?"
-    ]
-
-    for _ in range(min(10, len(sentences))):
-        sentence = random.choice(sentences)
-        template = random.choice(templates)
-        questions.append(template.format(sentence[:100]))  # limit to 100 chars
-
-    return "\n".join(questions)
+        return "No PDF uploaded."
+    return "Sample Question 1\nSample Question 2\nSample Question 3"
 
 # --------------------------
 # Gradio UI