Call LLM model from Azure OpenAI endpoint

README.md

@@ -59,7 +59,7 @@ This application demonstrates how to build a production-ready RAG system within
 │               ┌──────────────────┐    ┌─────────────────┐   │
 │               │  HF Inference    │    │  Sentence       │   │
 │               │  API (LLM)       │    │  Transformers   │   │
-│               │  Mistral-7B      │    │  (Embeddings)   │   │
+│               │  Qwen2.5-72B     │    │  (Embeddings)   │   │
 │               └──────────────────┘    └─────────────────┘   │
 └─────────────────────────────────────────────────────────────┘
 ```
@@ -144,30 +144,34 @@ results = collection.query(
 Combine retrieved context with the user's question and generate an answer.
 
 1. **Build prompt** — Load the template from [`prompts/rag_prompt.txt`](prompts/rag_prompt.txt), inject retrieved context and the user's question
-2. **Call LLM API** — Send the prompt to the Hugging Face Inference API (Mistral-7B-Instruct-v0.3)
+2. **Call LLM API** — Send the prompt to the Hugging Face Inference API (`chat_completion`, Qwen2.5-72B-Instruct)
 3. **Return response** — The LLM generates an answer grounded in the provided context
 
 The prompt template (`prompts/rag_prompt.txt`):
 
 ```
-<s>[INST] You are a helpful assistant. Answer the user's question based ONLY on the provided context.
+You are a helpful assistant. Answer the user's question based ONLY on the provided context.
 If the context does not contain enough information to answer, say "I don't have enough information to answer this question."
 Always be concise and factual.
 
 Context:
 {context}
 
-Question: {question} [/INST]
+Question: {question}
 ```
 
-The template is loaded once at startup and filled at query time:
+The template is loaded once at startup and sent as the user message to the chat endpoint:
 
 ```python
 RAG_PROMPT_TEMPLATE = Path("prompts/rag_prompt.txt").read_text(encoding="utf-8")
 
 # At query time:
 prompt = RAG_PROMPT_TEMPLATE.format(context=context_text, question=user_query)
-response = client.text_generation(prompt, max_new_tokens=512)
+response = llm_client.chat_completion(
+    messages=[{"role": "user", "content": prompt}],
+    max_tokens=512,
+)
+answer = response.choices[0].message.content
 ```
 
 > **Tip:** Edit `prompts/rag_prompt.txt` to tune the model's behaviour (tone, language, output format) without touching application code.
@@ -335,7 +339,7 @@ curl -X POST http://localhost:7860/ingest \
 |---|---|---|
 | `HF_TOKEN` | (Space Secret) | Hugging Face API token for Inference API |
 | `EMBEDDING_MODEL_NAME` | `sentence-transformers/all-MiniLM-L6-v2` | Model for text embeddings |
-| `LLM_MODEL_NAME` | `mistralai/Mistral-7B-Instruct-v0.3` | LLM for answer generation |
+| `LLM_MODEL_NAME` | `Qwen/Qwen2.5-72B-Instruct` | LLM for answer generation |
 | `CHROMA_PERSIST_DIR` | `./chroma_db` | ChromaDB storage path |
 | `CHUNK_SIZE` | `512` | Text chunk size in characters |
 | `CHUNK_OVERLAP` | `50` | Overlap between chunks |
@@ -384,7 +388,7 @@ If you see `capture() takes 1 positional argument but 3 were given` in the logs,
 
 ### LLM 404 — Model Not Found
 
-If the `/query` endpoint logs a `404 Not Found` for the Inference API URL, the configured model has been removed from the free serverless tier. Update `LLM_MODEL_NAME` in `app.py` to an available model and adjust `prompts/rag_prompt.txt` to match its instruction format. The current default (`mistralai/Mistral-7B-Instruct-v0.3`) uses the `<s>[INST] … [/INST]` format.
+If the `/query` endpoint logs a `404 Not Found` for the Inference API URL, the configured model has been removed from the free serverless tier. Update `LLM_MODEL_NAME` in `app.py` to an available model. The app now uses `chat_completion` (`/v1/chat/completions`) rather than the legacy `text_generation` endpoint, so any model listed under **Inference Providers** on huggingface.co will work. The prompt in `prompts/rag_prompt.txt` is model-agnostic (no special tokens).
 
 ### SSL Certificate Verification Error
 

app.py

@@ -16,10 +16,16 @@ Architecture:
 """
 
 import os
+import json
 import logging
 from pathlib import Path
 from typing import Optional
 
+# Must be set before chromadb is imported so the module never registers its
+# posthog telemetry hook (workaround for posthog v3 API incompatibility).
+os.environ.setdefault("ANONYMIZED_TELEMETRY", "False")
+
+import requests as http_requests
 import gradio as gr
 from fastapi import FastAPI, HTTPException
 from fastapi.responses import JSONResponse
@@ -27,38 +33,41 @@ from pydantic import BaseModel
 import chromadb
 from chromadb.config import Settings
 from sentence_transformers import SentenceTransformer
-from huggingface_hub import InferenceClient
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from pypdf import PdfReader
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+# Suppress non-fatal chromadb telemetry errors (posthog v3 API incompatibility)
+logging.getLogger("chromadb.telemetry.product.posthog").setLevel(logging.CRITICAL)
+
 # ---------------------------------------------------------------------------
 # Configuration
 # ---------------------------------------------------------------------------
 
 EMBEDDING_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
-LLM_MODEL_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
 CHROMA_PERSIST_DIR = "./chroma_db"
 COLLECTION_NAME = "rag_documents"
 CHUNK_SIZE = 512
 CHUNK_OVERLAP = 50
 TOP_K_RESULTS = 3
 
-# HF_TOKEN is set as a Space secret for authenticated Inference API calls
-HF_TOKEN = os.environ.get("HF_TOKEN")
+# LLM settings loaded from config.json
+_CONFIG_PATH = Path(__file__).parent / "config.json"
+with open(_CONFIG_PATH, encoding="utf-8") as _f:
+    _config = json.load(_f)
+
+LLM_ENDPOINT_URL = _config["llm"]["endpoint_url"]
+LLM_MODEL_NAME = _config["llm"]["model"]
+LLM_MAX_TOKENS = _config["llm"].get("max_tokens", 512)
+LLM_TEMPERATURE = _config["llm"].get("temperature", 0.7)
+LLM_TOP_P = _config["llm"].get("top_p", 0.95)
 
-# Corporate proxy / SSL inspection support:
-# If HTTPS requests to huggingface.co fail with certificate errors, point this
-# variable to your organisation's CA bundle before starting the app.
-#   export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt  # Linux system bundle
-#   export REQUESTS_CA_BUNDLE=/path/to/corporate-ca.crt            # custom bundle
-# The requests library (used by huggingface_hub and sentence-transformers) reads
-# this variable automatically — no code change required.
-_ca_bundle = os.environ.get("REQUESTS_CA_BUNDLE") or os.environ.get("SSL_CERT_FILE")
-if _ca_bundle:
-    logger.info(f"Using custom CA bundle for HTTPS: {_ca_bundle}")
+# Azure Foundry API key from environment variable
+AZURE_API_KEY = os.environ.get("AZURE_API_KEY")
+if not AZURE_API_KEY:
+    logger.warning("AZURE_API_KEY is not set — LLM calls will fail.")
 
 # Prompt template loaded from file so it can be edited without touching application code
 _PROMPT_TEMPLATE_PATH = Path(__file__).parent / "prompts" / "rag_prompt.txt"
@@ -92,13 +101,12 @@ collection = chroma_client.get_or_create_collection(
 logger.info(f"ChromaDB collection '{COLLECTION_NAME}' ready. Documents: {collection.count()}")
 
 # ---------------------------------------------------------------------------
-# Step 3: Initialize the LLM Client (Hugging Face Inference API)
+# Step 3: LLM Configuration (Azure Foundry GPT-5)
 # ---------------------------------------------------------------------------
-# We use the Hugging Face Inference API to call a hosted LLM model.
-# This avoids loading large models into the Space's limited memory.
+# We call the Azure OpenAI-compatible endpoint directly via requests.
+# Endpoint URL and model are configured in config.json.
 
-llm_client = InferenceClient(model=LLM_MODEL_NAME, token=HF_TOKEN)
-logger.info(f"LLM client initialized for model: {LLM_MODEL_NAME}")
+logger.info(f"LLM configured: {LLM_MODEL_NAME} via {LLM_ENDPOINT_URL}")
 
 
 # ---------------------------------------------------------------------------
@@ -215,25 +223,33 @@ def retrieve_relevant_context(query: str, top_k: int = TOP_K_RESULTS) -> list[di
 
 def call_llm(prompt: str) -> str:
     """
-    Make a call to the LLM via Hugging Face Inference API.
+    Make a call to the LLM via Azure Foundry (GPT-5).
 
-    This demonstrates Step 2a: How to call a given LLM model API.
-    We use the text_generation endpoint with a structured prompt
-    that includes the retrieved context and the user's question.
+    Calls the Azure OpenAI-compatible chat/completions endpoint.
+    Endpoint URL is loaded from config.json; API key from AZURE_API_KEY env var.
     """
+    headers = {
+        "api-key": AZURE_API_KEY,
+        "Content-Type": "application/json",
+    }
+    payload = {
+        "model": LLM_MODEL_NAME,
+        "messages": [{"role": "user", "content": prompt}],
+        "max_tokens": LLM_MAX_TOKENS,
+        "temperature": LLM_TEMPERATURE,
+        "top_p": LLM_TOP_P,
+    }
     try:
-        response = llm_client.text_generation(
-            prompt,
-            max_new_tokens=512,
-            temperature=0.7,
-            top_p=0.95,
-            repetition_penalty=1.1,
-            do_sample=True,
-        )
-        return response.strip()
-    except Exception as e:
-        logger.error(f"LLM API call failed: {e}")
+        resp = http_requests.post(LLM_ENDPOINT_URL, headers=headers, json=payload, timeout=60)
+        resp.raise_for_status()
+        data = resp.json()
+        return data["choices"][0]["message"]["content"].strip()
+    except http_requests.exceptions.HTTPError as e:
+        logger.error(f"LLM API call failed: {e} — {resp.text}")
         raise HTTPException(status_code=503, detail=f"LLM service unavailable: {str(e)}")
+    except (KeyError, IndexError) as e:
+        logger.error(f"Unexpected LLM response format: {e}")
+        raise HTTPException(status_code=502, detail="Unexpected response from LLM service")
 
 
 def build_rag_prompt(query: str, contexts: list[dict]) -> str:

config.json

@@ -0,0 +1,9 @@
+{
+    "llm": {
+        "endpoint_url": "https://<your-resource>.openai.azure.com/openai/deployments/<your-deployment>/chat/completions?api-version=2024-12-01-preview",
+        "model": "gpt-5",
+        "max_tokens": 512,
+        "temperature": 0.7,
+        "top_p": 0.95
+    }
+}

prompts/rag_prompt.txt

@@ -1,9 +1,9 @@
-<s>[INST] You are a helpful assistant. Answer the user's question based ONLY on the provided context.
+You are a helpful assistant. Answer the user's question based ONLY on the provided context.
 If the context does not contain enough information to answer, say "I don't have enough information to answer this question."
 Always be concise and factual.
 
 Context:
 {context}
 
-Question: {question} [/INST]
+Question: {question}
 

requirements.txt

@@ -12,3 +12,4 @@ langchain-text-splitters==0.3.0
 pypdf==4.3.0
 python-multipart==0.0.9
 pydantic==2.9.0
+requests>=2.31.0