{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "cb2ff14b", "metadata": {}, "outputs": [], "source": [ "import sys\n", "sys.path.append('..')" ] }, { "cell_type": "code", "execution_count": 2, "id": "6bb3bb7d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "f:\\Dissertation\\prod-rag-chat\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "from datasets import Dataset\n", "from app import retrieve_relevant_documents, emb_text, model, embedding_model\n", "from langchain_classic.chains.combine_documents import create_stuff_documents_chain\n", "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n", "from langchain_core.runnables import RunnableLambda\n", "from langchain_core.documents import Document" ] }, { "cell_type": "code", "execution_count": 3, "id": "e572fb31", "metadata": {}, "outputs": [], "source": [ "def setup_standalone_rag_chain():\n", " \"\"\"Setup a standalone RAG chain for testing without Chainlit session.\"\"\"\n", " \n", " def get_context_and_history(inputs):\n", " \"\"\"Retrieve context without session history.\"\"\"\n", " query = inputs[\"question\"]\n", " relevant_docs = retrieve_relevant_documents(query, limit=5)\n", " print(\"Relevant documents:\", relevant_docs[0] if relevant_docs else \"No documents found\")\n", " \n", " # Convert dictionaries to Document objects for LangChain\n", " doc_objects = []\n", " for doc in relevant_docs:\n", " doc_obj = Document(\n", " page_content=doc.get('text', ''),\n", " metadata=doc.get('metadata', {})\n", " )\n", " doc_objects.append(doc_obj)\n", "\n", " return {\n", " \"question\": query,\n", " \"context\": doc_objects,\n", " \"history\": [] # Empty history for testing\n", " }\n", " \n", " system_prompt = \"\"\"You are a helpful assistant specialising in developing non-discriminatory competence standards and disability support, reasonable adjustments, and equality legislation.\n", "\n", "When answering questions, you should:\n", "1. Use the provided context documents to inform your response\n", "2. Be accurate and helpful\n", "3. If the context doesn't contain relevant information, say so clearly\n", "4. Always reply in English\n", "5. Provide clear recommendations and examples wherever applicable\n", "6. Do not make assumptions about the user's knowledge or background\n", "7. If the user asks for a specific law or regulation, provide a brief explanation and cite relevant documents if available.\n", "8. Do not overemphasize disability in your responses, but rather focus on the support and adjustments that can be made to ensure equality and inclusivity.\n", "9. If the user query explicitly asks for a disability-related topic, provide a well-informed response based on the context documents.\n", "\n", "Context documents:\n", "{context} \n", "\n", "Please provide a clear response using the above context\n", "\"\"\"\n", "\n", " prompt = ChatPromptTemplate.from_messages([\n", " (\"system\", system_prompt),\n", " MessagesPlaceholder(variable_name=\"history\"),\n", " (\"human\", \"{question}\"),\n", " ])\n", "\n", " question_answer_chain = create_stuff_documents_chain(model, prompt)\n", " \n", " # Use a custom chain that properly handles our context and history\n", " def process_input_and_format(inputs):\n", " context_data = get_context_and_history(inputs)\n", " return {\n", " \"context\": context_data[\"context\"],\n", " \"question\": context_data[\"question\"],\n", " \"history\": context_data[\"history\"]\n", " }\n", " \n", " chain = RunnableLambda(process_input_and_format) | question_answer_chain\n", " \n", " return chain" ] }, { "cell_type": "code", "execution_count": 4, "id": "330ee35d", "metadata": {}, "outputs": [], "source": [ "\n", "# Setup the RAG chain\n", "rag_chain = setup_standalone_rag_chain()\n", "\n", "questions = [\"What are Provisions, Criteria and Practices?\", \n", " \"What is 'reasonable'?\",\n", " \"What is 'substantial disadvantage'?\",\n", " ]\n", "ground_truths = [\n", " \"\"\"The Equality and Human Rights Commission (EHRC) interprets PCPs as including:3 \n", "+ arrangements (for example, for deciding who to admit) \n", "+ the way that education, or access to any benefit, service or facility is offered or provided \n", "+ one-off or discretionary decisions \n", "+ proposals or directions to do something in a particular way \n", "+ formal and informal policies \n", "+ rules\"\"\",\n", "\n", " \"\"\"There are two key considerations of 'reasonableness' which can help when thinking through \n", "when an adjustment may be reasonable:4 \n", "+ Could the adjustment be practicable in its application (is it possible)? \n", "+ Could the adjustment be effective in achieving its aim (will it work)? \n", "There is no need to prove that the adjustment is practicable and effective in advance, just \n", "that it might be. An adjustment should not be considered unreasonable if it does not remove \n", "the disadvantage fully; an adjustment which partially removes or reduces substantial \n", "disadvantage is also likely to be reasonable.\"\"\",\n", "\n", " \"\"\"'Substantial' is defined in the Act as 'more than minor or trivial'. \n", "Examples of disadvantage recognised by the EHRC include: \n", "+ The additional time and effort expended by a disabled student \n", "+ The inconvenience, indignity, discomfort, or perceived disadvantage suffered by a \n", "disabled student \n", "+ The loss of opportunity or diminished progress experienced by a disabled student. \"\"\"]\n", "\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 5, "id": "ba3810dd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2025-12-20 19:42:27 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n", "Relevant documents: {'text': 'What is a provision, criterion or practice? The phrase ‘provision, criterion or practice’ is not defined by the Act. These concepts should be construed widely so as to include, for example, any formal or informal policies, rules, practices, arrangements, criteria, procedures, activities or provisions. They can cover one-off decisions and actions. In simple terms, they are about the way an education provider does things. Example:', 'metadata': {'source': 'data\\\\technical-guidance-further-higher-education.docx', 'file_directory': 'data', 'filename': 'technical-guidance-further-higher-education.docx', 'last_modified': '2025-07-02T21:00:50', 'page_number': 95, 'languages': ['eng'], 'filetype': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'category': 'CompositeElement', 'element_id': '3ae881ad6f81487213a9e234debf0921'}, 'score': 0.7807720899581909}\n", "2025-12-20 19:42:29 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n", "2025-12-20 19:42:32 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n", "2025-12-20 19:42:32 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n", "Relevant documents: {'text': '‘Reasonable’ means having regard to all of the circumstances including the nature of the act and how obviously discriminatory it is, the authority of the person making the statement and the knowledge that the helper has or ought to have.', 'metadata': {'source': 'data\\\\technical-guidance-further-higher-education.docx', 'file_directory': 'data', 'filename': 'technical-guidance-further-higher-education.docx', 'last_modified': '2025-07-02T21:00:50', 'page_number': 36, 'languages': ['eng'], 'filetype': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'category': 'CompositeElement', 'element_id': 'c5e3a60e2a6ccc88e0eff961f645a962'}, 'score': 0.7367081046104431}\n", "2025-12-20 19:42:33 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n", "2025-12-20 19:42:36 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n", "2025-12-20 19:42:37 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n", "Relevant documents: {'text': 'The Act states that disadvantage must be substantial, which is defined as more than minor or trivial. Whether such a disadvantage exists in a particular case is a question of fact, and is assessed on an objective basis. s212(1)', 'metadata': {'source': 'data\\\\technical-guidance-further-higher-education.docx', 'file_directory': 'data', 'filename': 'technical-guidance-further-higher-education.docx', 'last_modified': '2025-07-02T21:00:50', 'page_number': 89, 'languages': ['eng'], 'filetype': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'category': 'CompositeElement', 'element_id': 'b9e8ef04daf9150c9f7e32736b53df5b'}, 'score': 0.8376985788345337}\n", "2025-12-20 19:42:38 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n", "2025-12-20 19:42:40 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n" ] } ], "source": [ "answers = []\n", "contexts = []\n", "\n", "def clean_answer(answer):\n", " \"\"\"Remove tags and content from the answer.\"\"\"\n", " import re\n", " # Remove everything between and tags, including the tags themselves\n", " cleaned = re.sub(r'.*?\\s*', '', answer, flags=re.DOTALL)\n", " return cleaned.strip()\n", "\n", "# Inference\n", "for query in questions:\n", " # Get answer from the RAG chain\n", " answer = rag_chain.invoke({\"question\": query})\n", " # Clean the answer to remove thinking content\n", " cleaned_answer = clean_answer(answer)\n", " answers.append(cleaned_answer)\n", " \n", " # Get relevant documents for context\n", " relevant_docs = retrieve_relevant_documents(query, limit=5)\n", " context_texts = [doc['text'] for doc in relevant_docs]\n", " contexts.append(context_texts)\n", "\n", "# To dict\n", "data = {\n", " \"question\": questions,\n", " \"answer\": answers,\n", " \"contexts\": contexts,\n", " \"reference\": ground_truths\n", "}\n", "\n", "# Convert dict to dataset\n", "dataset = Dataset.from_dict(data)\n", "\n" ] }, { "cell_type": "code", "execution_count": 6, "id": "25c00c40", "metadata": {}, "outputs": [], "source": [ "from langchain_nebius import ChatNebius\n", "from pydantic import SecretStr\n", "import os" ] }, { "cell_type": "code", "execution_count": 7, "id": "2994b26e", "metadata": {}, "outputs": [], "source": [ "eval_model = ChatNebius(\n", " model=\"Qwen/Qwen3-235B-A22B-Instruct-2507\",\n", " streaming=False, # <--- CRITICAL: Must be False for Ragas\n", " temperature=0, # Lower temperature for the judge is better\n", " max_tokens=8192,\n", " top_p=0.95,\n", " api_key=SecretStr(os.getenv(\"OPENAI_API_KEY\")),\n", ")" ] }, { "cell_type": "code", "execution_count": 8, "id": "3e016be2", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Evaluating: 0%| | 0/12 [00:00