Update app.py

app.py

@@ -230,14 +230,12 @@ from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
 from llama_index.core import StorageContext, load_index_from_storage, VectorStoreIndex, SimpleDirectoryReader, ChatPromptTemplate, Settings
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.llms.huggingface_api import HuggingFaceInferenceAPI
 from pydantic import BaseModel
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.templating import Jinja2Templates
-from huggingface_hub import InferenceClient
-import json
 import datetime
 from simple_salesforce import Salesforce, SalesforceLogin
-from llama_index.llms.huggingface import HuggingFaceLLM
 
 # Pydantic model for request body
 class MessageRequest(BaseModel):
@@ -279,7 +277,7 @@ async def add_security_headers(request: Request, call_next):
 # CORS configuration
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["*"],  # Consider specifying allowed origins in production
+    allow_origins=["*"],  # Specify allowed origins in production
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
@@ -290,13 +288,11 @@ app.mount("/static", StaticFiles(directory="static"), name="static")
 templates = Jinja2Templates(directory="static")
 
 # LlamaIndex configuration
-Settings.llm = HuggingFaceLLM(
+Settings.llm = HuggingFaceInferenceAPI(
     model_name="meta-llama/Meta-Llama-3-8B-Instruct",
-    tokenizer_name="meta-llama/Meta-Llama-3-8B-Instruct",
-    context_window=3000,
     token=os.getenv("HF_TOKEN"),
     max_new_tokens=512,
-    generate_kwargs={"temperature": 0.1},
+    temperature=0.1
 )
 
 Settings.embed_model = HuggingFaceEmbedding(