モデルを変えてみた

app.py

@@ -49,8 +49,12 @@ index = faiss.IndexFlatIP(dimension)  # Cosine用にnormalize済ならこれ
 index.add(np.array(doc_embeddings))
 
 def generate_text(prompt):
-    result = llm(search(prompt))
-    return result['choices'][0]['text']
+    full_prompt = search(prompt)
+    input_ids = tokenizer(full_prompt, return_tensors="pt").input_ids.cuda()  # または .to("cuda") / .to("mps") / .to("cpu") に変更
+    with torch.no_grad():
+        output_ids = llm.generate(input_ids, max_new_tokens=256)
+    result_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+    return result_text
 
 def search(query):
     query_embedding = model.encode([query], normalize_embeddings=True)