Update main.py

main.py

@@ -1,15 +1,14 @@
 import os
+import re
 import asyncio
 import logging
-import json
-import torch
-import numpy as np
-import httpx
-import uvicorn
-
 from contextlib import asynccontextmanager
 from urllib.parse import urlparse
 
+import numpy as np
+import torch
+import uvicorn
+
 from bs4 import BeautifulSoup
 from trafilatura import extract
 from ddgs import DDGS
@@ -19,11 +18,71 @@ from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 
 from pydantic import BaseModel
-from sentence_transformers import SentenceTransformer
+from sentence_transformers import SentenceTransformer, CrossEncoder
 from transformers import (
     AutoTokenizer,
     AutoModelForCausalLM,
-    AutoConfig
+    AutoConfig,
+)
+
+
+# ============================================================
+# CONFIGURATION
+# ============================================================
+
+MODEL_NAME = os.getenv(
+    "MODEL_NAME",
+    "Qwen/Qwen2.5-7B-Instruct"
+)
+
+EMBED_MODEL_NAME = os.getenv(
+    "EMBED_MODEL_NAME",
+    "sentence-transformers/all-MiniLM-L6-v2"
+)
+
+RERANKER_MODEL_NAME = os.getenv(
+    "RERANKER_MODEL_NAME",
+    "BAAI/bge-reranker-base"
+)
+
+MAX_HISTORY_MESSAGES = int(
+    os.getenv("MAX_HISTORY_MESSAGES", "8")
+)
+
+MAX_REASONING_EXAMPLES = int(
+    os.getenv("MAX_REASONING_EXAMPLES", "4")
+)
+
+MAX_WEB_RESULTS = int(
+    os.getenv("MAX_WEB_RESULTS", "5")
+)
+
+MAX_WEB_CHUNKS = int(
+    os.getenv("MAX_WEB_CHUNKS", "6")
+)
+
+WEB_CHUNK_WORDS = int(
+    os.getenv("WEB_CHUNK_WORDS", "500")
+)
+
+WEB_CHUNK_OVERLAP = int(
+    os.getenv("WEB_CHUNK_OVERLAP", "75")
+)
+
+MAX_NEW_TOKENS = int(
+    os.getenv("MAX_NEW_TOKENS", "2048")
+)
+
+MAX_INPUT_TOKENS = int(
+    os.getenv("MAX_INPUT_TOKENS", "12000")
+)
+
+RAG_MIN_SIMILARITY = float(
+    os.getenv("RAG_MIN_SIMILARITY", "0.28")
+)
+
+RERANK_MIN_SCORE = float(
+    os.getenv("RERANK_MIN_SCORE", "-5.0")
 )
 
 
@@ -33,7 +92,7 @@ from transformers import (
 
 logging.basicConfig(
     level=logging.INFO,
-    format="%(asctime)s - %(levelname)s - %(message)s"
+    format="%(asctime)s - %(levelname)s - %(message)s",
 )
 
 logger = logging.getLogger("infinitygrm")
@@ -45,45 +104,44 @@ logger = logging.getLogger("infinitygrm")
 
 engine_state = {}
 
-# Temporary in-memory session storage
-# Replace with Redis later if needed.
 session_store = {}
 
+reasoning_embeddings = None
+
 
 # ============================================================
 # REASONING EXAMPLES
 #
-# These are intentionally concise.
+# These examples teach STRUCTURES and PATTERNS.
 #
-# The model should learn:
-# - how to identify the problem type
-# - how to reject tempting wrong approaches
-# - how to identify useful state/invariants
-# - how to verify an answer
-# - how to reason about complexity
+# They are NOT required for the model to answer.
 #
-# You can expand this list to 200-500+ examples.
+# The model is explicitly instructed to solve novel problems
+# from first principles when no relevant example exists.
 # ============================================================
 
 REASONING_EXAMPLES = [
 
     {
-        "category": "algorithms",
+        "category": "arrays",
+        "subcategories": [
+            "prefix_sum",
+            "hashmap",
+            "subarray",
+        ],
         "problem": """
 Given an array of integers that may contain negative numbers,
 find the longest contiguous subarray whose sum equals k.
 """,
         "reasoning": """
-Key observation:
-Negative numbers make a normal sliding window unreliable because
-adding an element can decrease the sum and removing an element can
-increase it.
+Key observation: negative numbers make a normal sliding window
+unreliable because the sum is not monotonic.
 
-Use prefix sums.
+Use prefix sums. If the current prefix sum is S, an earlier prefix
+sum of S-k means the elements between those positions sum to k.
 
-If the current prefix sum is S, we need an earlier prefix sum S-k.
-Store the earliest index of every prefix sum because the earliest
-index gives the longest possible subarray.
+Store the earliest index at which each prefix sum appears because
+the earliest occurrence produces the longest possible subarray.
 """,
         "answer": """
 Use a hashmap of prefix sums.
@@ -92,234 +150,275 @@ prefix = {0: -1}
 running_sum = 0
 best = 0
 
-For each index i:
+For each index:
     running_sum += nums[i]
 
     if running_sum - k exists:
-        best = max(best, i - prefix[running_sum - k])
+        update best
 
-    store running_sum only if it has not been seen before.
+    store running_sum only if it has not appeared before.
 
 Time: O(n) average.
 Space: O(n).
-"""
+""",
     },
 
     {
         "category": "graphs",
+        "subcategories": [
+            "shortest_path",
+            "state_expansion",
+            "dijkstra",
+        ],
         "problem": """
 A directed graph has positive edge costs. You may use a discount
 exactly once, causing one chosen edge to cost floor(cost / 2).
 Find the minimum cost from s to t.
 """,
         "reasoning": """
-The cost of traversing an edge depends on whether the discount has
-already been used.
+The important property is that whether the discount has already been
+used changes the future choices.
 
-Therefore one distance per node is insufficient.
+Therefore node alone is not a sufficient state.
 
-Use two states:
-(node, unused)
-(node, used)
+Represent the state as:
+(node, discount_unused)
+(node, discount_used)
 
-From an unused state, an edge can either be taken normally or with
-the discount. From a used state, it must be taken normally.
+From the unused state, an edge can be traversed normally or with the
+discount. From the used state, it must be traversed normally.
 
-Run Dijkstra on this expanded state graph.
+Because all resulting costs are non-negative, Dijkstra can be run on
+this expanded state graph.
 """,
         "answer": """
-Dijkstra over 2*n states.
-
-unused -> unused with cost c
-unused -> used with cost c // 2
-used -> used with cost c
+Run Dijkstra over the expanded state space.
 
-Answer is the distance to t in the appropriate final state.
-
-Time: O((n+m) log n).
-Space: O(n+m).
-"""
+Time: O((V + E) log V).
+Space: O(V + E).
+""",
     },
 
     {
-        "category": "algorithms",
+        "category": "graphs",
+        "subcategories": [
+            "cycle_detection",
+            "dfs",
+            "directed_graph",
+        ],
         "problem": """
 Determine whether a directed graph contains a cycle in O(V+E).
 """,
         "reasoning": """
-A visited set alone is insufficient for directed-cycle detection.
-
-A node can be completely processed without forming a cycle, while
-another edge can point back to a node currently being explored.
+A simple visited set is insufficient for directed-cycle detection.
 
 Use three states:
 0 = unvisited
 1 = currently exploring
 2 = completely processed
 
-An edge to a state-1 node is a back edge and proves a cycle.
+During DFS, an edge to a state-1 vertex is a back edge and proves the
+existence of a cycle.
+
+A state-2 vertex does not indicate a cycle in the current DFS path.
 """,
         "answer": """
 Use DFS with three colors/states.
 
-Time: O(V+E).
+Time: O(V + E).
 Space: O(V).
-"""
+""",
     },
 
     {
         "category": "data_structures",
+        "subcategories": [
+            "hashmap",
+            "frequency",
+            "strings",
+        ],
         "problem": """
 Find the first non-repeating character in a string.
 """,
         "reasoning": """
 The answer depends on both frequency and original order.
 
-First count every character.
-Then scan the string again and return the first character whose
-frequency is one.
+First count every character. Then scan the string again and return
+the first character whose frequency is one.
 """,
         "answer": """
 Use a frequency hashmap followed by a second scan.
 
 Time: O(n).
 Space: O(k), where k is the number of distinct characters.
-"""
+""",
     },
 
     {
-        "category": "algorithms",
+        "category": "arrays",
+        "subcategories": [
+            "two_pointers",
+            "sorted_array",
+        ],
         "problem": """
 Given a sorted array, determine whether two numbers sum to target.
 """,
         "reasoning": """
-Because the array is sorted, two pointers can eliminate many
-possibilities at once.
+The sorted order lets us eliminate entire ranges of candidates.
 
-If left + right is too small, increase left.
-If it is too large, decrease right.
-If equal, the target exists.
+If the sum is too small, move the left pointer right.
+If the sum is too large, move the right pointer left.
+If it equals target, the pair exists.
 """,
         "answer": """
 Use two pointers.
 
 Time: O(n).
 Space: O(1).
-"""
+""",
     },
 
     {
-        "category": "algorithms",
+        "category": "graphs",
+        "subcategories": [
+            "bfs",
+            "unweighted_shortest_path",
+        ],
         "problem": """
 Find the shortest path in an unweighted graph.
 """,
         "reasoning": """
 Every edge has equal cost.
 
-Therefore BFS explores nodes in increasing number of edges from
-the source. The first time a node is reached is its shortest
-distance from the source.
+BFS explores vertices by increasing number of edges from the source.
+Therefore the first time a vertex is reached, its distance is
+minimal.
 """,
         "answer": """
 Use BFS with a queue.
 
-Time: O(V+E).
+Time: O(V + E).
 Space: O(V).
-"""
+""",
     },
 
     {
-        "category": "algorithms",
+        "category": "graphs",
+        "subcategories": [
+            "dijkstra",
+            "weighted_shortest_path",
+        ],
         "problem": """
 Find the shortest path in a graph with positive weighted edges.
 """,
         "reasoning": """
 BFS is not sufficient because edges have different costs.
 
-Since all edge weights are positive, Dijkstra's greedy selection
-of the currently cheapest unsettled node is valid.
+With positive weights, Dijkstra's greedy choice is valid: the
+currently smallest unsettled distance cannot later be improved by a
+path containing additional non-negative edges.
 """,
         "answer": """
 Use Dijkstra with a min-heap.
 
 Typical complexity:
-O((V+E) log V).
-"""
+O((V + E) log V).
+""",
     },
 
     {
-        "category": "algorithms",
+        "category": "graphs",
+        "subcategories": [
+            "0_1_bfs",
+            "shortest_path",
+        ],
         "problem": """
 A graph has only edge weights 0 and 1. Find shortest paths.
 """,
         "reasoning": """
-Ordinary BFS treats every edge as having equal cost, which is wrong.
+Ordinary BFS treats all edges equally, which is incorrect.
 
-Dijkstra works, but a deque can exploit the special weights.
+Dijkstra works, but the special weights allow a deque:
+zero-cost edges go to the front and one-cost edges go to the back.
 
-When traversing a zero-cost edge, add the node to the front.
-When traversing a one-cost edge, add it to the back.
+This preserves increasing distance order without a heap.
 """,
         "answer": """
 Use 0-1 BFS.
 
-Time: O(V+E).
+Time: O(V + E).
 Space: O(V).
-"""
+""",
     },
 
     {
         "category": "dynamic_programming",
+        "subcategories": [
+            "linear_dp",
+            "state_transition",
+        ],
         "problem": """
 Given positive integers, maximize the sum without selecting adjacent
 elements.
 """,
         "reasoning": """
-At every position there are two choices:
+At each element there are two choices.
 
-Skip the current element.
-Take the current element, which means the previous element cannot
-be taken.
+Skip it:
+dp[i] = dp[i-1]
 
-This gives:
+Take it:
+dp[i] = dp[i-2] + nums[i]
+
+Therefore:
 
 dp[i] = max(dp[i-1], dp[i-2] + nums[i])
 
-Only the previous two values are required.
+Only the previous two states are required.
 """,
         "answer": """
-Use dynamic programming with O(1) extra space.
+Use dynamic programming with two rolling values.
 
 Time: O(n).
 Space: O(1).
-"""
+""",
     },
 
     {
         "category": "binary_search",
+        "subcategories": [
+            "search_on_answer",
+            "monotonicity",
+        ],
         "problem": """
 Find the minimum integer x for which a feasibility condition becomes
 true.
 """,
         "reasoning": """
-Binary search is appropriate only if feasibility is monotonic.
+Binary search on the answer is valid only when feasibility is
+monotonic.
 
-If x is feasible, and every larger x is also feasible, then the
-search space has a false -> true boundary.
+If x is feasible, every larger value must also be feasible, creating
+a false-to-true boundary.
 
-Search for the first true value.
+Search for the first feasible value.
 """,
         "answer": """
 Use binary search on the answer.
 
-At every midpoint, evaluate feasibility.
+Perform one feasibility check per midpoint.
 
-Time is O(log C) feasibility checks, where C is the search range.
-"""
+Complexity:
+O(log C) feasibility checks, where C is the search range.
+""",
     },
 
     {
         "category": "algorithm_selection",
+        "subcategories": [
+            "binary_search",
+            "monotonicity",
+        ],
         "problem": """
 Should binary search automatically be used whenever an answer is a
 number?
@@ -327,21 +426,24 @@ number?
         "reasoning": """
 No.
 
-Binary search requires an ordered or monotonic search condition.
-The fact that the answer is numeric is not enough.
+A numeric answer is not sufficient.
 
-First establish that a feasibility predicate has a monotonic
-true/false boundary.
+Binary search requires an ordered search space or a monotonic
+predicate that creates a usable boundary.
 """,
         "answer": """
 Do not automatically use binary search.
 
-Check monotonicity first.
-"""
+Prove monotonicity first.
+""",
     },
 
     {
         "category": "debugging",
+        "subcategories": [
+            "control_flow",
+            "python",
+        ],
         "problem": """
 A function returns the first unique character but returns None too
 early.
@@ -356,18 +458,22 @@ Find the bug.
         "reasoning": """
 The return None statement is inside the loop.
 
-Therefore the function stops after checking only the first character
-that is not unique.
+Therefore the function terminates after checking only the first
+non-unique character.
 
-The failure is control flow, not the counting logic.
+The counting logic is not necessarily wrong; the control flow is.
 """,
         "answer": """
 Move return None outside the loop.
-"""
+""",
     },
 
     {
         "category": "complexity",
+        "subcategories": [
+            "nested_work",
+            "list_membership",
+        ],
         "problem": """
 What is the complexity of:
 
@@ -377,32 +483,36 @@ for x in values:
         result.append(x)
 """,
         "reasoning": """
-Membership testing in a Python list is O(n).
+Membership testing in a Python list is O(n) in the worst case.
 
-That operation occurs for each input element.
+That operation can occur once for every input element.
 
-Therefore the worst-case complexity is O(n²).
+Therefore the overall worst-case complexity is O(n²).
 """,
         "answer": """
 Worst case:
+
 Time: O(n²)
 Space: O(n)
 
-A set can reduce average membership checking to O(1).
-"""
+A set can provide average O(1) membership testing.
+""",
     },
 
     {
         "category": "optimization",
+        "subcategories": [
+            "hash_set",
+            "deduplication",
+        ],
         "problem": """
 Remove duplicates from a list while preserving original order.
 """,
         "reasoning": """
-We need fast membership testing and must preserve the first
-occurrence.
+The result must preserve the first occurrence while membership
+testing needs to be fast.
 
-A set provides fast membership testing while the result list
-preserves order.
+Use a set for membership and a list for output ordering.
 """,
         "answer": """
 seen = set()
@@ -415,11 +525,15 @@ for x in values:
 
 Average time: O(n).
 Space: O(n).
-"""
+""",
     },
 
     {
         "category": "stacks",
+        "subcategories": [
+            "parentheses",
+            "matching",
+        ],
         "problem": """
 Determine whether parentheses, brackets, and braces are correctly
 balanced and nested.
@@ -429,120 +543,125 @@ The most recently opened bracket must be the first bracket closed.
 
 That is last-in-first-out behavior, so a stack is appropriate.
 
-When opening a bracket, push it.
-When closing one, compare it with the stack top.
+Opening brackets are pushed. Closing brackets must match the stack
+top.
 """,
         "answer": """
 Use a stack.
 
 Time: O(n).
 Space: O(n).
-"""
+""",
     },
 
     {
         "category": "sliding_window",
+        "subcategories": [
+            "strings",
+            "two_pointers",
+        ],
         "problem": """
 Find the longest substring without repeating characters.
 """,
         "reasoning": """
 Maintain a window containing unique characters.
 
-When a repeated character appears, move the left boundary beyond
-its previous occurrence.
+When a duplicate appears, move the left boundary beyond the previous
+occurrence of that character.
 
-The right pointer only moves forward, so each character is processed
-a constant number of times.
+Both boundaries only move forward.
 """,
         "answer": """
-Use a sliding window and hashmap.
+Use a sliding window with a hashmap.
 
 Time: O(n).
 Space: O(k).
-"""
+""",
     },
 
     {
-        "category": "sorting",
+        "category": "selection",
+        "subcategories": [
+            "quickselect",
+            "order_statistics",
+        ],
         "problem": """
 Find the kth largest element without fully sorting the array.
 """,
         "reasoning": """
-Full sorting costs O(n log n), but the problem only needs one order
-statistic.
+Full sorting performs more work than necessary because only one order
+statistic is needed.
 
-Quickselect partitions around a pivot and only recursively processes
-the side containing the desired position.
+Quickselect partitions around a pivot and continues only on the side
+containing the desired position.
 """,
         "answer": """
-Quickselect has average O(n) time and O(1) auxiliary partition
-space, depending on implementation.
-"""
-    },
+Use Quickselect.
 
-    {
-        "category": "graphs",
-        "problem": """
-Determine whether an undirected graph is connected.
-""",
-        "reasoning": """
-Starting from any vertex, perform DFS or BFS.
-
-If every vertex is reached, the graph is connected.
-Otherwise, at least one vertex belongs to another component.
+Average time: O(n).
+Worst-case time: O(n²) without appropriate pivot guarantees.
 """,
-        "answer": """
-DFS or BFS.
-
-Time: O(V+E).
-Space: O(V).
-"""
     },
 
     {
         "category": "trees",
+        "subcategories": [
+            "dfs",
+            "recursion",
+        ],
         "problem": """
 Find the maximum depth of a binary tree.
 """,
         "reasoning": """
 The depth of a node is one plus the maximum depth of its children.
 
-This naturally forms a recursive recurrence:
-depth(node) = 1 + max(depth(left), depth(right)).
+This gives the recurrence:
+
+depth(node) =
+    1 + max(depth(left), depth(right))
 """,
         "answer": """
-Recursive DFS.
+Use recursive DFS.
 
 Time: O(n).
-Space: O(h), where h is tree height.
-"""
+Space: O(h), where h is the tree height.
+""",
     },
 
     {
         "category": "trees",
+        "subcategories": [
+            "postorder",
+            "height",
+            "balance",
+        ],
         "problem": """
 Determine whether a binary tree is height-balanced.
 """,
         "reasoning": """
-Checking the height independently for every node can repeatedly
+Computing the height independently for every node can repeatedly
 traverse the same subtrees.
 
-Instead, calculate height bottom-up.
+Calculate heights bottom-up instead.
 
-Return failure immediately when a subtree is unbalanced.
+Return failure immediately if a subtree is unbalanced.
 """,
         "answer": """
 Use postorder DFS.
 
-Return height or a sentinel indicating imbalance.
+Return the subtree height or a sentinel indicating imbalance.
 
 Time: O(n).
 Space: O(h).
-"""
+""",
     },
 
     {
         "category": "greedy",
+        "subcategories": [
+            "proof",
+            "exchange_argument",
+        ],
         "problem": """
 Can you always solve an optimization problem greedily by taking the
 locally cheapest option?
@@ -550,90 +669,108 @@ locally cheapest option?
         "reasoning": """
 No.
 
-A local choice may prevent a better global solution.
+A local choice can prevent a better global solution.
 
 A greedy algorithm needs a proof that the local choice is safe,
-usually through an exchange argument or another greedy property.
+often through an exchange argument or another structural property.
 """,
         "answer": """
 Do not assume greedy works.
-Find a counterexample or prove the greedy choice is always safe.
-"""
+
+Prove the greedy choice is safe or construct a counterexample.
+""",
     },
 
     {
         "category": "algorithm_selection",
+        "subcategories": [
+            "state_design",
+            "dynamic_programming",
+        ],
         "problem": """
-You have a problem where a state can be reached in multiple ways,
-and future options depend on how that state was reached. Should you
-store only one distance per node?
+A state can be reached multiple ways, and future options depend on
+how the state was reached. Should you store only one distance per
+node?
 """,
         "reasoning": """
 Not necessarily.
 
-If the future behavior depends on additional information, that
+If future behavior depends on additional information, that
 information is part of the state.
 
-For example, whether a discount has been used creates two distinct
-states even when the physical node is identical.
+For example, whether a discount has been used can distinguish two
+otherwise identical graph nodes.
 """,
         "answer": """
 Expand the state representation.
 
-Use (node, extra_state) rather than only node.
-"""
+Use something like:
+(node, extra_state)
+instead of only:
+node
+""",
     },
 
     {
         "category": "verification",
+        "subcategories": [
+            "testing",
+            "edge_cases",
+        ],
         "problem": """
 How should a newly written algorithm be tested?
 """,
         "reasoning": """
-Do not test only the normal case.
+Normal examples are insufficient.
 
-Use:
-- smallest valid input
+Test:
+- smallest valid inputs
 - empty input when allowed
-- duplicate values
+- duplicates
 - negative values
 - boundary values
-- already sorted/reversed data
-- cases where the answer is impossible
-- cases with multiple optimal answers
-- large inputs for complexity
+- sorted and reverse-sorted data
+- impossible cases
+- multiple optimal answers
+- very large inputs
+- adversarial structures
 """,
         "answer": """
-Test normal cases, edge cases, adversarial cases, and large cases.
-"""
+Use normal, edge-case, adversarial, and large-input tests.
+""",
     },
 
     {
         "category": "complexity",
+        "subcategories": [
+            "nested_algorithms",
+        ],
         "problem": """
-An algorithm performs a binary search and runs Dijkstra for every
+An algorithm performs binary search and runs Dijkstra for every
 binary-search step. What happens to the complexity?
 """,
         "reasoning": """
-The costs multiply.
+Nested algorithm costs multiply.
 
 If Dijkstra costs O(E log V) and binary search performs O(log C)
 iterations, the total is:
 
 O(E log V log C)
-
-It cannot simply be described as O(E log V).
 """,
         "answer": """
-Multiply nested algorithm costs.
+Multiply the costs of nested operations.
 
 Total:
 O(E log V log C).
-"""
+""",
     },
 
     {
-        "category": "coding",
+        "category": "intervals",
+        "subcategories": [
+            "sorting",
+            "merging",
+        ],
         "problem": """
 Write a Python function that merges overlapping intervals.
 """,
@@ -643,139 +780,172 @@ Sort intervals by starting point.
 After sorting, the next interval either overlaps the current interval
 or starts after it ends.
 
-If it overlaps, extend the current end.
-Otherwise, finalize the current interval and start a new one.
+When they overlap, extend the current endpoint. Otherwise finalize
+the current interval and start a new one.
 """,
         "answer": """
-Sort first, then scan.
+Sort by interval start, then scan.
 
 Time: O(n log n).
 Space: O(n) for the output.
-"""
+""",
     },
 
     {
-        "category": "coding",
+        "category": "strings",
+        "subcategories": [
+            "two_pointers",
+            "normalization",
+        ],
         "problem": """
-Given a string, determine whether it is a palindrome after removing
+Determine whether a string is a palindrome after removing
 non-alphanumeric characters and ignoring case.
 """,
         "reasoning": """
-Normalization can be performed while comparing characters from both
-ends.
+Compare characters from both ends while skipping non-alphanumeric
+characters.
+
+Normalize characters before comparison.
 
-Skip characters that are not alphanumeric.
-Compare normalized characters.
+This avoids constructing a second normalized string.
 """,
         "answer": """
-Two pointers.
+Use two pointers.
 
 Time: O(n).
-Space: O(1) if normalization is performed in-place during scanning.
-"""
+Auxiliary space: O(1).
+""",
     },
 
     {
         "category": "debugging",
+        "subcategories": [
+            "recursion",
+            "stack",
+        ],
         "problem": """
 A recursive function works on small inputs but crashes on very large
 inputs.
 """,
         "reasoning": """
-The algorithm may be correct, but Python recursion has a recursion
-depth limitation.
+The algorithm may be logically correct but exceed Python's recursion
+depth.
+
+Determine whether recursion depth grows with input size.
 
-First determine whether recursion depth is proportional to input
-size. If so, consider an iterative stack or another formulation.
+If so, an iterative formulation using an explicit stack may avoid the
+language recursion limit.
 """,
         "answer": """
-Check recursion depth.
-Convert to iteration when necessary.
-"""
+Check recursion depth and convert to iteration when appropriate.
+""",
     },
 
     {
         "category": "graphs",
+        "subcategories": [
+            "negative_weights",
+            "shortest_path",
+        ],
         "problem": """
 Why can't ordinary Dijkstra handle negative edge weights?
 """,
         "reasoning": """
-Dijkstra relies on the fact that once the smallest tentative
-distance is selected, no later path can improve it.
+Dijkstra assumes that once the smallest tentative distance is
+selected, no future path can improve it.
 
-A negative edge can produce a cheaper path later, violating that
-assumption.
+A negative edge can create a cheaper path later, violating that
+greedy assumption.
 """,
         "answer": """
-Negative edges invalidate Dijkstra's greedy correctness condition.
-Consider Bellman-Ford or another algorithm appropriate to the graph.
-"""
+Negative weights invalidate Dijkstra's correctness condition.
+
+Consider Bellman-Ford or another algorithm suitable for the input.
+""",
     },
 
     {
         "category": "math",
+        "subcategories": [
+            "bitwise",
+            "powers_of_two",
+        ],
         "problem": """
 How can you determine whether an integer is a power of two?
 """,
         "reasoning": """
-A positive power of two has exactly one bit set in binary.
+A positive power of two has exactly one bit set.
 
 For positive n:
+
 n & (n - 1) == 0
-exactly when n is a power of two.
+
+is true exactly when n has one set bit.
 """,
         "answer": """
 Check:
 
 n > 0 and (n & (n - 1)) == 0
-"""
+""",
     },
 
     {
         "category": "algorithm_selection",
+        "subcategories": [
+            "complexity",
+            "optimization",
+        ],
         "problem": """
-You are given a problem with n up to 200,000. A proposed solution
+You are given a problem where n can be 200,000. A proposed solution
 uses O(n²). What should you investigate?
 """,
         "reasoning": """
-At n = 200,000, quadratic work is generally too large.
+Quadratic work is generally too large at this input scale.
 
-Look for a structural improvement:
-hashing, sorting, two pointers, prefix sums, binary search,
-divide-and-conquer, a heap, graph algorithms, or dynamic programming.
+Look for structural improvements such as hashing, sorting, two
+pointers, prefix sums, binary search, heaps, graph algorithms, or
+dynamic programming.
 """,
         "answer": """
-Inspect the O(n²) bottleneck and search for an O(n log n) or O(n)
+Identify the O(n²) bottleneck and search for an O(n log n) or O(n)
 formulation.
-"""
+""",
     },
 
     {
         "category": "problem_solving",
+        "subcategories": [
+            "debugging",
+            "edge_cases",
+        ],
         "problem": """
-A solution works on the examples but fails hidden tests. What
-should you investigate first?
+A solution works on examples but fails hidden tests. What should be
+investigated?
 """,
         "reasoning": """
-Check assumptions.
+Check unstated assumptions.
 
-Typical causes:
-incorrect boundary handling,
+Typical causes include:
+incorrect boundaries,
 empty inputs,
-duplicate values,
-integer overflow,
+duplicates,
+overflow,
 incorrect state transitions,
 wrong complexity,
-or an algorithm that only works under unstated assumptions.
+or an algorithm that only works on the sample structure.
 """,
         "answer": """
 Use adversarial and edge-case testing and verify the algorithm's
 invariants.
-"""
+""",
     },
 
     {
         "category": "invariants",
+        "subcategories": [
+            "correctness",
+            "proof",
+        ],
         "problem": """
 What is an invariant in an algorithm?
 """,
@@ -783,41 +953,49 @@ What is an invariant in an algorithm?
 An invariant is a property that remains true throughout the relevant
 part of the algorithm.
 
-Identifying one helps prove correctness and detect implementation
+It can be used both to prove correctness and to detect implementation
 errors.
 """,
         "answer": """
-State the property that must remain true after every iteration or
-state transition.
-"""
+State the property that must remain true after every relevant
+iteration or state transition.
+""",
     },
 
     {
-        "category": "dp",
+        "category": "dynamic_programming",
+        "subcategories": [
+            "state_design",
+            "transitions",
+        ],
         "problem": """
 How do you design a dynamic programming solution to an unfamiliar
 problem?
 """,
         "reasoning": """
-Start by identifying the smallest information needed to describe a
+Start by identifying the smallest information required to describe a
 subproblem.
 
 Then define:
 1. state
 2. transition
 3. base cases
-4. order of computation
+4. computation order
 5. final state
 
-Avoid creating a state containing unnecessary information.
+Avoid putting unnecessary information into the state.
 """,
         "answer": """
-Define the state and transition before writing the implementation.
-"""
+Define the state and transition before implementing the DP.
+""",
     },
 
     {
-        "category": "debugging",
+        "category": "optimization",
+        "subcategories": [
+            "profiling",
+            "complexity",
+        ],
         "problem": """
 A program produces the right answer but is too slow. What should
 you do?
@@ -828,34 +1006,43 @@ Correctness and efficiency are separate properties.
 First identify the dominant operation using complexity analysis or
 profiling.
 
-Then replace the bottleneck rather than randomly optimizing code.
+Then improve the actual bottleneck instead of randomly optimizing
+unrelated code.
 """,
         "answer": """
 Find the dominant bottleneck, determine its complexity, and improve
-the algorithm or data structure responsible.
-"""
+the responsible algorithm or data structure.
+""",
     },
 
     {
-        "category": "coding",
+        "category": "arrays",
+        "subcategories": [
+            "prefix_sum",
+            "hashmap",
+        ],
         "problem": """
 Why should a dictionary store the earliest index of a prefix sum when
 finding the longest subarray with a target sum?
 """,
         "reasoning": """
-For a fixed ending index i, a smaller earlier index creates a longer
+For a fixed ending index, a smaller earlier index creates a longer
 subarray.
 
-Therefore replacing an existing prefix-sum index with a later index
-can only make future answers shorter.
+Replacing an existing prefix-sum index with a later index can only
+make future answers shorter.
 """,
         "answer": """
-Keep the earliest occurrence of each prefix sum.
-"""
+Keep the earliest occurrence of every prefix sum.
+""",
     },
 
     {
         "category": "logic",
+        "subcategories": [
+            "implication",
+            "affirming_consequent",
+        ],
         "problem": """
 A statement says: If A is true, then B is true. Does B being true
 prove A is true?
@@ -863,59 +1050,46 @@ prove A is true?
         "reasoning": """
 No.
 
-The implication only establishes A -> B.
+The implication establishes A -> B, not B -> A.
 
-B may be true for other reasons. Inferring A from B is affirming the
-consequent.
+B may be true for another reason.
 """,
         "answer": """
 No. A -> B does not imply B -> A.
-"""
-    },
-
-    {
-        "category": "coding",
-        "problem": """
-A Python function needs to process a stream containing millions of
-items, but all items cannot fit comfortably in memory. What should
-you consider?
-""",
-        "reasoning": """
-Avoid loading the entire stream at once.
-
-Process incrementally, keeping only the state needed to produce the
-answer.
-
-This is a streaming algorithm.
 """,
-        "answer": """
-Process items one at a time or in bounded batches.
-"""
     },
 
     {
         "category": "systems",
+        "subcategories": [
+            "gpu_inference",
+            "concurrency",
+            "batching",
+        ],
         "problem": """
 Twenty users may request inference from the same GPU-hosted language
 model simultaneously. How can the server avoid exhausting GPU
 memory?
 """,
         "reasoning": """
-Do not allow unlimited concurrent generation.
+Model weights can be shared, but generation memory grows with active
+requests and sequence lengths.
 
-Use a request queue, concurrency limits, and preferably batching
-where supported.
+Unlimited concurrency can therefore exhaust memory.
 
-The model weights can be shared, while generation memory grows with
-active requests and sequence lengths.
+Use request queuing, bounded concurrency, and controlled batching.
 """,
         "answer": """
 Use bounded concurrency, request queuing, and controlled batching.
-"""
+""",
     },
 
     {
-        "category": "reasoning",
+        "category": "correctness",
+        "subcategories": [
+            "assumptions",
+            "proof",
+        ],
         "problem": """
 An algorithm's proof assumes all edge weights are positive, but the
 implementation accepts negative weights. Is the proof still valid?
@@ -923,32 +1097,162 @@ implementation accepts negative weights. Is the proof still valid?
         "reasoning": """
 No.
 
-The implementation's input domain is broader than the proof's
-assumptions.
+The implementation accepts inputs outside the assumptions used by
+the proof.
 
-Correctness guarantees only apply when the stated assumptions hold.
+Correctness guarantees only apply to inputs satisfying those
+assumptions.
 """,
         "answer": """
-No. Either enforce the positive-weight constraint or use an algorithm
+Either enforce the positive-weight constraint or use an algorithm
 whose correctness supports negative weights.
-"""
-    }
-
+""",
+    },
 ]
 
 
 # ============================================================
-# EMBEDDING CACHE
+# CATEGORY ROUTING
 # ============================================================
 
-reasoning_embeddings = None
+CATEGORY_HINTS = {
+    "graphs": [
+        "graph",
+        "graphs",
+        "vertex",
+        "vertices",
+        "edge",
+        "edges",
+        "shortest path",
+        "dijkstra",
+        "bfs",
+        "dfs",
+        "topological",
+        "cycle",
+        "connected",
+    ],
+
+    "dynamic_programming": [
+        "dynamic programming",
+        "dp",
+        "subproblem",
+        "transition",
+        "memoization",
+        "tabulation",
+    ],
+
+    "binary_search": [
+        "binary search",
+        "monotonic",
+        "first true",
+        "last true",
+        "search on answer",
+    ],
+
+    "sliding_window": [
+        "sliding window",
+        "substring",
+        "subarray",
+        "window",
+    ],
+
+    "trees": [
+        "binary tree",
+        "tree",
+        "bst",
+        "subtree",
+        "node",
+        "leaf",
+    ],
+
+    "debugging": [
+        "bug",
+        "debug",
+        "error",
+        "crash",
+        "wrong answer",
+        "fails",
+        "broken",
+    ],
+
+    "complexity": [
+        "complexity",
+        "big o",
+        "time complexity",
+        "space complexity",
+        "too slow",
+        "performance",
+    ],
+
+    "arrays": [
+        "array",
+        "arrays",
+        "subarray",
+        "prefix sum",
+        "two pointers",
+    ],
+
+    "strings": [
+        "string",
+        "substring",
+        "character",
+        "palindrome",
+    ],
+
+    "systems": [
+        "gpu",
+        "cuda",
+        "server",
+        "concurrency",
+        "batching",
+        "memory",
+        "inference",
+        "api",
+    ],
+}
+
+
+def detect_categories(query: str) -> set:
+    q = query.lower()
+
+    categories = set()
+
+    for category, keywords in CATEGORY_HINTS.items():
+        for keyword in keywords:
+
+            if " " in keyword:
+                if keyword in q:
+                    categories.add(category)
+                    break
+            else:
+                if re.search(
+                    rf"\b{re.escape(keyword)}\b",
+                    q,
+                ):
+                    categories.add(category)
+                    break
+
+    return categories
+
+
+# ============================================================
+# EMBEDDINGS
+# ============================================================
+
+def example_to_embedding_text(example: dict) -> str:
+
+    return (
+        f"CATEGORY: {example['category']}\n"
+        f"SUBCATEGORIES: "
+        f"{', '.join(example.get('subcategories', []))}\n"
+        f"PROBLEM: {example['problem']}\n"
+        f"REASONING: {example['reasoning']}\n"
+        f"ANSWER: {example['answer']}"
+    )
 
 
 def build_reasoning_embeddings():
-    """
-    Build embeddings once during startup so we don't repeatedly
-    encode the entire example library for every user request.
-    """
+
     global reasoning_embeddings
 
     if not REASONING_EXAMPLES:
@@ -960,37 +1264,28 @@ def build_reasoning_embeddings():
     if embed_model is None:
         return
 
-    texts = []
-
-    for example in REASONING_EXAMPLES:
-        combined = (
-            example["category"]
-            + "\n"
-            + example["problem"]
-            + "\n"
-            + example["reasoning"]
-        )
-        texts.append(combined)
+    texts = [
+        example_to_embedding_text(example)
+        for example in REASONING_EXAMPLES
+    ]
 
     reasoning_embeddings = embed_model.encode(
         texts,
         normalize_embeddings=True,
-        convert_to_numpy=True
+        convert_to_numpy=True,
+        show_progress_bar=False,
     )
 
     logger.info(
-        f"Loaded {len(REASONING_EXAMPLES)} reasoning examples."
+        "Built embeddings for %d reasoning examples.",
+        len(REASONING_EXAMPLES),
     )
 
 
 def retrieve_reasoning_examples(
     query: str,
-    top_k: int = 4
+    top_k: int = 4,
 ) -> list:
-    """
-    Retrieve only the most relevant examples instead of inserting
-    the entire example library into every prompt.
-    """
 
     global reasoning_embeddings
 
@@ -1003,50 +1298,120 @@ def retrieve_reasoning_examples(
         return REASONING_EXAMPLES[:top_k]
 
     try:
+
+        if reasoning_embeddings is None:
+            build_reasoning_embeddings()
+
         query_embedding = embed_model.encode(
             [query],
             normalize_embeddings=True,
-            convert_to_numpy=True
+            convert_to_numpy=True,
+            show_progress_bar=False,
         )[0]
 
-        if reasoning_embeddings is None:
-            build_reasoning_embeddings()
-
         scores = np.dot(
             reasoning_embeddings,
-            query_embedding
+            query_embedding,
         )
 
-        indices = np.argsort(scores)[::-1][:top_k]
+        ranked_indices = np.argsort(scores)[::-1]
+
+        query_categories = detect_categories(query)
 
         selected = []
+        selected_indices = set()
+
+        # ----------------------------------------------------
+        # First pass: category-compatible examples
+        # ----------------------------------------------------
+
+        for index in ranked_indices:
+
+            index = int(index)
+
+            example = REASONING_EXAMPLES[index]
+
+            score = float(scores[index])
+
+            if score < RAG_MIN_SIMILARITY:
+                continue
 
-        for index in indices:
-            example = REASONING_EXAMPLES[int(index)].copy()
-            example["_similarity"] = float(scores[int(index)])
-            selected.append(example)
+            if (
+                query_categories
+                and example["category"] not in query_categories
+            ):
+                continue
+
+            selected.append(
+                {
+                    **example,
+                    "_similarity": score,
+                }
+            )
+
+            selected_indices.add(index)
+
+            if len(selected) >= top_k:
+                break
+
+        # ----------------------------------------------------
+        # Second pass: semantic fallback
+        #
+        # This is crucial.
+        #
+        # If no category-specific example exists, semantic
+        # similarity can still find a structurally useful
+        # example.
+        # ----------------------------------------------------
+
+        if len(selected) < top_k:
+
+            for index in ranked_indices:
+
+                index = int(index)
+
+                if index in selected_indices:
+                    continue
+
+                score = float(scores[index])
+
+                if score < RAG_MIN_SIMILARITY:
+                    continue
+
+                example = REASONING_EXAMPLES[index]
+
+                selected.append(
+                    {
+                        **example,
+                        "_similarity": score,
+                    }
+                )
+
+                selected_indices.add(index)
+
+                if len(selected) >= top_k:
+                    break
 
         return selected
 
     except Exception as e:
+
         logger.warning(
-            f"Reasoning example retrieval failed: {e}"
+            "Reasoning retrieval failed: %s",
+            e,
         )
 
-        return REASONING_EXAMPLES[:top_k]
+        return []
 
 
 def format_reasoning_examples(
     query: str,
-    top_k: int = 4
+    top_k: int = 4,
 ) -> str:
-    """
-    Convert retrieved examples into compact prompt context.
-    """
 
     examples = retrieve_reasoning_examples(
         query,
-        top_k=top_k
+        top_k=top_k,
     )
 
     if not examples:
@@ -1060,14 +1425,16 @@ def format_reasoning_examples(
             f"""
 EXAMPLE {i}
 CATEGORY: {example["category"]}
+SUBCATEGORIES: {", ".join(example.get("subcategories", []))}
+RELEVANCE: {example["_similarity"]:.3f}
 
 PROBLEM:
 {example["problem"].strip()}
 
-REASONING PATTERN:
+APPROACH SUMMARY:
 {example["reasoning"].strip()}
 
-EXPECTED APPROACH:
+ANSWER:
 {example["answer"].strip()}
 """.strip()
         )
@@ -1076,433 +1443,90 @@ EXPECTED APPROACH:
 
 
 # ============================================================
-# MODEL INITIALIZATION
+# TEXT UTILITIES
 # ============================================================
 
-@asynccontextmanager
-async def lifespan(app: FastAPI):
-
-    logger.info(
-        "Initializing neural weights and engine dependencies..."
-    )
-
-    device = (
-        "cuda"
-        if torch.cuda.is_available()
-        else "cpu"
-    )
-
-    dtype = (
-        torch.float16
-        if device == "cuda"
-        else torch.float32
-    )
-
-    model_name = "Qwen/Qwen2.5-3B-Instruct"
-
-    logger.info(
-        f"Loading model: {model_name}"
-    )
-
-    tokenizer = AutoTokenizer.from_pretrained(
-        model_name
-    )
-
-    tokenizer.pad_token = tokenizer.eos_token
-
-    config = AutoConfig.from_pretrained(
-        model_name
-    )
-
-    config.pad_token_id = tokenizer.eos_token_id
-
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        config=config,
-        torch_dtype=dtype,
-        device_map="auto" if device == "cuda" else None
-    ).eval()
-
-    embed_model = SentenceTransformer(
-        "sentence-transformers/all-MiniLM-L6-v2",
-        device=device
-    )
-
-    engine_state["tokenizer"] = tokenizer
-    engine_state["model"] = model
-    engine_state["embed_model"] = embed_model
-
-    engine_state["max_history"] = 15
-
-    # Number of relevant examples inserted into each prompt.
-    engine_state["reasoning_examples"] = 4
-
-    logger.info(
-        f"Engine mapped to: {device}"
-    )
-
-    # Build example embeddings once.
-    build_reasoning_embeddings()
-
-    yield
-
-    engine_state.clear()
+def normalize_text(text: str) -> str:
 
-    logger.info(
-        "Application context fully flushed."
+    text = re.sub(
+        r"\s+",
+        " ",
+        text or "",
     )
 
+    return text.strip()
 
-# ============================================================
-# FASTAPI
-# ============================================================
-
-app = FastAPI(
-    title="Infinity GRM Engine",
-    lifespan=lifespan
-)
-
-
-# ============================================================
-# CORS
-# ============================================================
-
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
 
+def chunk_text(
+    text: str,
+    chunk_size: int = WEB_CHUNK_WORDS,
+    overlap: int = WEB_CHUNK_OVERLAP,
+) -> list:
 
-# ============================================================
-# REQUEST MODEL
-# ============================================================
-
-class ChatRequest(BaseModel):
-    message: str
-    session_id: str
-
-
-# ============================================================
-# SESSION MANAGEMENT
-# ============================================================
-
-def fetch_or_create_session(
-    session_id: str
-) -> dict:
-
-    if session_id in session_store:
-        return session_store[session_id]
-
-    default_session = {
-        "history": [],
-        "last_entity": "",
-        "last_web_query": "",
-        "last_response": "",
-        "last_web_context": []
-    }
-
-    session_store[session_id] = default_session
-
-    return default_session
-
-
-def save_session_to_redis(
-    session_id: str,
-    session: dict
-):
-    """
-    Temporary in-memory storage.
-
-    Replace with Redis later.
-    """
-
-    session_store[session_id] = session
-
-
-def update_history_buffer(
-    session: dict,
-    role: str,
-    content: str
-):
-
-    session["history"].append(
-        {
-            "role": role,
-            "content": content
-        }
-    )
-
-    if len(session["history"]) > engine_state["max_history"]:
-        session["history"].pop(0)
-
+    words = normalize_text(text).split()
 
-# ============================================================
-# SEMANTIC SIMILARITY
-# ============================================================
+    if not words:
+        return []
 
-def calculate_semantic_similarity(
-    a: str,
-    b: str
-) -> float:
+    chunks = []
 
-    embedder = engine_state["embed_model"]
+    start = 0
 
-    embeddings = embedder.encode(
-        [a, b],
-        normalize_embeddings=True
-    )
+    while start < len(words):
 
-    return float(
-        np.dot(
-            embeddings[0],
-            embeddings[1]
+        end = min(
+            start + chunk_size,
+            len(words),
         )
-    )
-
-
-# ============================================================
-# FOLLOW-UP DETECTION
-# ============================================================
-
-def checks_pronoun_reference(
-    query: str
-) -> bool:
-
-    pronoun_references = {
-        "he",
-        "him",
-        "his",
-        "she",
-        "her",
-        "hers",
-        "they",
-        "them",
-        "their",
-        "it",
-        "its",
-        "that",
-        "those",
-        "this",
-        "these",
-        "then",
-        "next",
-        "after"
-    }
-
-    return any(
-        word in pronoun_references
-        for word in query.lower().split()
-    )
-
 
-def is_followup(
-    query: str,
-    session: dict
-) -> bool:
-
-    if not session["last_entity"]:
-        return False
-
-    scores = []
-
-    try:
-        scores.append(
-            calculate_semantic_similarity(
-                query,
-                session["last_entity"]
-            )
+        chunk = " ".join(
+            words[start:end]
         )
-    except Exception:
-        pass
-
-    try:
-        if session["last_response"]:
-            scores.append(
-                calculate_semantic_similarity(
-                    query,
-                    session["last_response"][:500]
-                )
-            )
-    except Exception:
-        pass
-
-    best_match = max(
-        scores,
-        default=0
-    )
-
-    return (
-        best_match > 0.45
-        or len(query.split()) <= 3
-        or checks_pronoun_reference(query)
-    )
 
+        if chunk:
+            chunks.append(chunk)
 
-# ============================================================
-# LLM GENERATION
-# ============================================================
-
-def execute_llm_generation(
-    prompt: str,
-    max_new_tokens: int = 2048
-) -> str:
-
-    tokenizer = engine_state["tokenizer"]
-    model = engine_state["model"]
-
-    inputs = tokenizer(
-        prompt,
-        return_tensors="pt"
-    )
+        if end >= len(words):
+            break
 
-    inputs = {
-        key: value.to(model.device)
-        for key, value in inputs.items()
-    }
-
-    with torch.no_grad():
-
-        out = model.generate(
-            **inputs,
-            max_new_tokens=max_new_tokens,
-            temperature=0.15,
-            top_p=0.9,
-            repetition_penalty=1.08,
-            do_sample=True,
-            pad_token_id=tokenizer.eos_token_id
+        start = max(
+            start + 1,
+            end - overlap,
         )
 
-    generated_tokens = out[
-        0
-    ][
-        inputs["input_ids"].shape[1]:
-    ]
-
-    return tokenizer.decode(
-        generated_tokens,
-        skip_special_tokens=True
-    ).strip()
-
+    return chunks
 
-# ============================================================
-# TOPIC EXTRACTION
-# ============================================================
 
-def run_topic_extraction(
-    query: str
+def trim_text_by_tokens(
+    text: str,
+    max_tokens: int,
 ) -> str:
 
-    prompt = f"""
-Extract the primary subject of this question.
+    tokenizer = engine_state.get("tokenizer")
 
-Query:
-{query}
-
-Return only the subject.
-""".strip()
+    if tokenizer is None:
+        return text
 
-    return execute_llm_generation(
-        prompt,
-        max_new_tokens=64
-    ).split("\n")[0].strip()
-
-
-# ============================================================
-# FOLLOW-UP QUERY REWRITING
-# ============================================================
-
-def rewrite_followup_query(
-    query: str,
-    session: dict
-) -> str:
-
-    history = "\n".join(
-        [
-            f"User: {x['user']}\nSearch: {x['search']}"
-            for x in session["last_web_context"][-5:]
-        ]
+    tokens = tokenizer.encode(
+        text,
+        add_special_tokens=False,
     )
 
-    prompt = f"""
-Entity:
-{session["last_entity"]}
-
-Recent conversation:
-{history}
-
-User follow-up:
-{query}
-
-Rewrite the follow-up as a standalone web search query.
-
-Return only the search query.
-""".strip()
-
-    return execute_llm_generation(
-        prompt,
-        max_new_tokens=128
-    ).split("\n")[0].strip()
-
-
-# ============================================================
-# WEB SEARCH DETECTION
-# ============================================================
-
-def should_trigger_web_search(
-    q: str,
-    session: dict
-) -> bool:
-
-    q = q.lower()
-
-    search_triggers = [
-        "who",
-        "what",
-        "when",
-        "where",
-        "why",
-        "how",
-        "founder",
-        "ceo",
-        "company",
-        "price",
-        "stock",
-        "news",
-        "latest",
-        "today",
-        "current",
-        "recent",
-        "update",
-        "code",
-        "python",
-        "html",
-        "javascript",
-        "documentation",
-        "release",
-        "version"
-    ]
-
-    if any(
-        trigger in q
-        for trigger in search_triggers
-    ):
-        return True
+    if len(tokens) <= max_tokens:
+        return text
 
-    return is_followup(
-        q,
-        session
+    return tokenizer.decode(
+        tokens[:max_tokens],
+        skip_special_tokens=True,
     )
 
 
 # ============================================================
-# DUCKDUCKGO
+# WEB SEARCH
 # ============================================================
 
 def isolated_blocking_ddg(
-    query_string: str
+    query_string: str,
 ) -> list:
 
     try:
@@ -1511,45 +1535,44 @@ def isolated_blocking_ddg(
 
             results = ddgs.text(
                 query_string,
-                max_results=5
+                max_results=MAX_WEB_RESULTS,
             )
 
             if results:
 
-                return [
-                    r["href"]
-                    for r in results
-                    if "href" in r
-                ]
+                urls = []
+
+                for result in results:
+
+                    href = result.get("href")
+
+                    if href:
+                        urls.append(href)
+
+                return list(
+                    dict.fromkeys(urls)
+                )
 
     except Exception as e:
 
         logger.error(
-            f"Official DuckDuckGo Search failed: {e}"
+            "DuckDuckGo search failed: %s",
+            e,
         )
 
     return []
 
 
 async def search_web_async(
-    query: str
+    query: str,
 ) -> list:
 
     urls = await asyncio.to_thread(
         isolated_blocking_ddg,
-        query
+        query,
     )
 
-    if not urls:
-
-        urls.append(
-            "https://en.wikipedia.org/wiki/"
-            + query.replace(" ", "_")
-        )
-
-    return list(
-        dict.fromkeys(urls)
-    )[:5]
+    return urls[:MAX_WEB_RESULTS]
 
 
 # ============================================================
@@ -1558,36 +1581,45 @@ async def search_web_async(
 
 async def scrape_target_url(
     client,
-    url: str
+    url: str,
 ):
 
     try:
 
-        r = await client.get(
+        response = await client.get(
             url,
-            timeout=10.0
+            timeout=12.0,
         )
 
-        if r.status_code != 200:
+        if response.status_code != 200:
 
             logger.warning(
-                f"HTTP {r.status_code}: {url}"
+                "HTTP %s: %s",
+                response.status_code,
+                url,
             )
 
             return None
 
+        html = response.text
+
         text = None
 
         try:
 
             text = extract(
-                r.text
+                html,
+                include_comments=False,
+                include_tables=True,
+                favor_precision=True,
             )
 
         except Exception as e:
 
             logger.warning(
-                f"Trafilatura failed for {url}: {e}"
+                "Trafilatura failed for %s: %s",
+                url,
+                e,
             )
 
         if not text:
@@ -1595,454 +1627,31 @@ async def scrape_target_url(
             try:
 
                 soup = BeautifulSoup(
-                    r.text,
-                    "html.parser"
+                    html,
+                    "html.parser",
                 )
 
+                for tag in soup(
+                    [
+                        "script",
+                        "style",
+                        "noscript",
+                        "nav",
+                        "footer",
+                        "header",
+                    ]
+                ):
+                    tag.decompose()
+
                 text = soup.get_text(
                     separator=" ",
-                    strip=True
+                    strip=True,
                 )
 
             except Exception as e:
 
-                logger.error(
-                    f"BeautifulSoup failed for {url}: {e}"
-                )
-
-                return None
-
-        if not text or len(text.split()) < 20:
-            return None
-
-        return {
-            "url": url,
-            "domain": urlparse(url).netloc,
-            "text": text[:2500]
-        }
-
-    except Exception as e:
-
-        logger.exception(
-            f"Scraping error for {url}: {e}"
-        )
-
-        return None
-
-
-# ============================================================
-# WEB CONTEXT PIPELINE
-# ============================================================
-
-async def build_context_pipeline(
-    query: str
-) -> tuple:
-
-    context_blocks = []
-    extraction_sources = []
-
-    target_urls = await search_web_async(
-        query
-    )
-
-    headers = {
-        "User-Agent":
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-            "AppleWebKit/537.36 "
-            "(KHTML, like Gecko) "
-            "Chrome/122.0.0.0 Safari/537.36",
-
-        "Accept":
-            "text/html,application/xhtml+xml,"
-            "application/xml;q=0.9,image/avif,"
-            "image/webp,*/*;q=0.8",
-
-        "Accept-Language":
-            "en-US,en;q=0.9",
-
-        "Accept-Encoding":
-            "gzip, deflate, br",
-
-        "Connection":
-            "keep-alive",
-
-        "Upgrade-Insecure-Requests":
-            "1"
-    }
-
-    from curl_cffi.requests import AsyncSession
-
-    async with AsyncSession(
-        impersonate="chrome",
-        headers=headers
-    ) as client:
-
-        tasks = [
-            scrape_target_url(
-                client,
-                u
-            )
-            for u in target_urls
-        ]
-
-        scraped_results = await asyncio.gather(
-            *tasks
-        )
-
-    for result in scraped_results:
-
-        if result:
-
-            context_blocks.append(
-                f"[{result['domain']}] "
-                f"{result['text']}"
-            )
-
-            extraction_sources.append(
-                result["url"]
-            )
-
-    return (
-        "\n\n".join(context_blocks),
-        list(set(extraction_sources))
-    )
-
-
-# ============================================================
-# MAIN ORCHESTRATION
-# ============================================================
-
-async def process_ask_orchestration(
-    q: str,
-    session: dict
-) -> tuple:
-
-    use_web = should_trigger_web_search(
-        q,
-        session
-    )
-
-    search_query = q
-
-    if (
-        use_web
-        and is_followup(q, session)
-        and session["last_entity"]
-    ):
-
-        search_query = rewrite_followup_query(
-            q,
-            session
-        )
-
-    context = ""
-    sources = []
-
-    # --------------------------------------------------------
-    # WEB SEARCH
-    # --------------------------------------------------------
-
-    if use_web:
-
-        if not is_followup(q, session):
-
-            try:
-
-                session["last_entity"] = (
-                    run_topic_extraction(q)
-                )
-
-            except Exception:
-
-                session["last_entity"] = q
-
-        context, sources = (
-            await build_context_pipeline(
-                search_query
-            )
-        )
-
-        if len(context.strip()) < 150:
-
-            context, sources = (
-                await build_context_pipeline(
-                    search_query + " wikipedia"
-                )
-            )
-
-        session[
-            "last_web_context"
-        ].append(
-            {
-                "user": q,
-                "search": search_query
-            }
-        )
-
-        if len(
-            session["last_web_context"]
-        ) > 5:
-
-            session[
-                "last_web_context"
-            ].pop(0)
-
-        session[
-            "last_web_query"
-        ] = search_query
-
-    # --------------------------------------------------------
-    # RETRIEVE REASONING EXAMPLES
-    # --------------------------------------------------------
-
-    reasoning_context = format_reasoning_examples(
-        q,
-        top_k=engine_state.get(
-            "reasoning_examples",
-            4
-        )
-    )
-
-    # --------------------------------------------------------
-    # SYSTEM PROMPT
-    # --------------------------------------------------------
-
-    system_prompt = """
-You are Infinity GRM, a capable reasoning and coding assistant.
-
-Solve problems carefully rather than jumping to the first familiar
-algorithm.
-
-For difficult problems:
-
-1. Identify exactly what the problem requires.
-2. Identify important constraints and assumptions.
-3. Determine the key property or structure.
-4. Consider tempting approaches when useful.
-5. Reject an approach if its assumptions do not hold.
-6. Choose the appropriate algorithm or data structure.
-7. Check important edge cases.
-8. Verify the result.
-9. Give the final answer clearly.
-
-For programming problems:
-- Prefer correct and efficient algorithms.
-- Respect the stated constraints.
-- Explain important implementation decisions.
-- State time and space complexity.
-- Do not claim an algorithm is O(n) if it contains a hidden
-  O(n log n), O(n²), or repeated expensive operation.
-
-Use the provided reasoning examples as patterns, not as facts that
-must be copied.
-
-If web context is provided, use it when relevant.
-Do not invent information that contradicts the provided context.
-
-Do not blindly follow an example if the current problem is different.
-""".strip()
-
-    # --------------------------------------------------------
-    # BUILD MESSAGES
-    # --------------------------------------------------------
-
-    messages = [
-        {
-            "role": "system",
-            "content": system_prompt
-        }
-    ]
-
-    # --------------------------------------------------------
-    # REASONING EXAMPLES
-    # --------------------------------------------------------
-
-    if reasoning_context:
-
-        messages.append(
-            {
-                "role": "system",
-                "content":
-                    "RELEVANT REASONING EXAMPLES:\n\n"
-                    + reasoning_context
-            }
-        )
-
-    # --------------------------------------------------------
-    # WEB CONTEXT
-    # --------------------------------------------------------
-
-    if use_web:
-
-        messages.extend(
-            session["history"][-10:]
-        )
-
-        messages.append(
-            {
-                "role": "user",
-                "content":
-                    f"WEB CONTEXT:\n"
-                    f"{context}\n\n"
-                    f"QUESTION:\n"
-                    f"{q}"
-            }
-        )
-
-    else:
-
-        messages.extend(
-            session["history"][-8:]
-        )
-
-        messages.append(
-            {
-                "role": "user",
-                "content": q
-            }
-        )
-
-    # --------------------------------------------------------
-    # CHAT TEMPLATE
-    # --------------------------------------------------------
-
-    compiled_prompt = (
-        engine_state["tokenizer"]
-        .apply_chat_template(
-            messages,
-            tokenize=False,
-            add_generation_prompt=True
-        )
-    )
-
-    # --------------------------------------------------------
-    # GENERATE
-    # --------------------------------------------------------
-
-    response_string = execute_llm_generation(
-        compiled_prompt,
-        max_new_tokens=2048
-    )
-
-    session[
-        "last_response"
-    ] = response_string
-
-    update_history_buffer(
-        session,
-        "user",
-        q
-    )
-
-    update_history_buffer(
-        session,
-        "assistant",
-        response_string
-    )
-
-    return (
-        response_string,
-        sources
-    )
-
-
-# ============================================================
-# API ENDPOINTS
-# ============================================================
-
-@app.post("/query")
-async def chat_endpoint(
-    request: ChatRequest
-):
-
-    try:
-
-        session = fetch_or_create_session(
-            request.session_id
-        )
-
-        answer, sources = (
-            await process_ask_orchestration(
-                request.message,
-                session
-            )
-        )
-
-        save_session_to_redis(
-            request.session_id,
-            session
-        )
-
-        return {
-            "response": answer,
-            "sources": sources
-        }
-
-    except Exception as e:
-
-        logger.exception(
-            "Critical exception inside chat runtime pipeline"
-        )
-
-        raise HTTPException(
-            status_code=500,
-            detail=str(e)
-        )
-
-
-@app.get("/health")
-async def validation_heartbeat():
-
-    return {
-        "status": "healthy",
-        "gpu_acceleration_active":
-            torch.cuda.is_available(),
-        "reasoning_examples":
-            len(REASONING_EXAMPLES)
-    }
-
-
-# ============================================================
-# STATIC FILES
-# ============================================================
-
-if not os.path.exists("static"):
-    os.makedirs("static")
-
-
-app.mount(
-    "/static",
-    StaticFiles(directory="static"),
-    name="static"
-)
-
-
-@app.get("/")
-async def home_route_processor():
-
-    return {
-        "message":
-            "Infinity GRM Engine Core is active.",
-        "reasoning_examples":
-            len(REASONING_EXAMPLES)
-    }
-
-
-# ============================================================
-# HUGGING FACE PORT
-# ============================================================
-
-if __name__ == "__main__":
-
-    port = int(
-        os.getenv(
-            "PORT",
-            7860
-        )
-    )
-
-    uvicorn.run(
-        "main:app",
-        host="0.0.0.0",
-        port=port
-    )
+                logger.warning(
+                    "BeautifulSoup failed for %s: %s",
+                    url,
+                    e,
+                )