Update conversation_logic.py

conversation_logic.py

@@ -1,42 +1,119 @@
 from __future__ import annotations
 
-from typing import Any, Dict, List, Optional
+import re
+from typing import Any, Dict, List, Optional, Set
 
+from context_parser import detect_intent, intent_to_help_mode
 from formatting import format_reply
 from generator_engine import GeneratorEngine
 from models import RetrievedChunk, SolverResult
 from quant_solver import is_quant_question, solve_quant
 from retrieval_engine import RetrievalEngine
-
+from utils import short_lines
+
+
+# -----------------------------
+# Retrieval intent configuration
+# -----------------------------
+
+RETRIEVAL_ALLOWED_INTENTS = {
+    "walkthrough",
+    "step_by_step",
+    "explain",
+    "method",
+    "hint",
+    "definition",
+    "concept",
+    "instruction",
+}
+
+DIRECT_SOLVE_PATTERNS = [
+    r"\bsolve\b",
+    r"\bwhat is\b",
+    r"\bfind\b",
+    r"\bgive (?:me )?the answer\b",
+    r"\bjust the answer\b",
+    r"\banswer only\b",
+    r"\bcalculate\b",
+]
+
+STRUCTURE_KEYWORDS = {
+    "algebra": [
+        "equation", "solve", "isolate", "variable", "linear", "expression",
+        "unknown", "algebra", "substitute", "rearrange"
+    ],
+    "percent": [
+        "percent", "%", "percentage", "increase", "decrease", "of"
+    ],
+    "ratio": [
+        "ratio", "proportion", "proportional", "part", "share"
+    ],
+    "statistics": [
+        "mean", "median", "mode", "range", "average", "standard deviation"
+    ],
+    "probability": [
+        "probability", "chance", "likely", "odds", "event"
+    ],
+    "geometry": [
+        "triangle", "circle", "angle", "area", "perimeter", "radius", "diameter"
+    ],
+    "number_properties": [
+        "integer", "odd", "even", "prime", "divisible", "factor", "multiple"
+    ],
+}
+
+INTENT_KEYWORDS = {
+    "walkthrough": ["walkthrough", "work through", "step by step", "full working"],
+    "step_by_step": ["step", "first step", "next step", "step by step"],
+    "explain": ["explain", "why", "understand"],
+    "method": ["method", "approach", "how do i solve", "how to solve"],
+    "hint": ["hint", "nudge", "clue"],
+    "definition": ["define", "definition", "what does", "what is meant by"],
+    "concept": ["concept", "idea", "principle", "rule"],
+    "instruction": ["how do i", "how to", "what should i do first", "what step"],
+}
+
+MISMATCH_TERMS = {
+    "algebra": [
+        "absolute value", "modulus", "square root", "quadratic", "inequality",
+        "roots", "parabola", "simultaneous equations"
+    ],
+    "percent": [
+        "triangle", "circle", "prime", "absolute value"
+    ],
+    "ratio": [
+        "absolute value", "quadratic", "circle"
+    ],
+    "statistics": [
+        "absolute value", "prime", "triangle"
+    ],
+    "probability": [
+        "absolute value", "circle area", "quadratic"
+    ],
+    "geometry": [
+        "absolute value", "prime", "median salary"
+    ],
+    "number_properties": [
+        "circle", "triangle", "absolute value"
+    ],
+}
+
+
+# -----------------------------
+# Reply building
+# -----------------------------
 
 def _teaching_lines(chunks: List[RetrievedChunk]) -> List[str]:
-    lines: List[str] = []
+    lines = []
     for chunk in chunks:
-        text = (chunk.text or "").strip().replace("\n", " ")
+        text = chunk.text.strip().replace("\n", " ")
         if len(text) > 220:
             text = text[:217].rstrip() + "…"
-        topic = chunk.topic or "general"
+        topic = getattr(chunk, "topic", "general") or "general"
         lines.append(f"- {topic}: {text}")
     return lines
 
 
-def _should_retrieve(intent: str, result: SolverResult) -> bool:
-    if not result.solved:
-        return True
-    return intent in {"hint", "method", "step_by_step", "full_working", "walkthrough", "explain"}
-
-
-def _retrieval_query(user_text: str, question_text: str, options_text: str) -> str:
-    parts = []
-    if question_text.strip():
-        parts.append(question_text.strip())
-    if options_text.strip():
-        parts.append(options_text.strip())
-    if user_text.strip():
-        parts.append(user_text.strip())
-    return "\n".join(parts).strip()
-
-
 def _compose_quant_reply(
     result: SolverResult,
     intent: str,
@@ -44,128 +121,341 @@ def _compose_quant_reply(
     verbosity: float,
 ) -> str:
     steps = result.steps or []
-    internal = (
-        getattr(result, "internal_answer", None)
-        or getattr(result, "internal_answer_value", None)
-        or getattr(result, "answer_value", None)
-        or ""
-    )
+    internal = result.internal_answer or result.answer_value or ""
 
     if intent == "hint":
-        if steps:
-            return f"Hint:\n- {steps[0]}"
-        return "Hint:\n- Start by translating the wording into an equation."
+        return steps[0] if steps else "Start by translating the wording into an equation."
 
-    if intent in {"method", "walkthrough"}:
-        body = "Use this method:"
+    if intent == "instruction":
         if steps:
-            limit = 3 if verbosity < 0.75 else 4
-            body += "\n" + "\n".join(f"- {s}" for s in steps[:limit])
-        else:
-            body += "\n- Treat the statement as an equation.\n- Isolate the unknown step by step."
-        if reveal_answer and internal:
-            body += f"\n\nThat gives {internal}."
-        return body
+            return f"First step: {steps[0]}"
+        return "First, turn the wording into a mathematical relationship."
 
-    if intent in {"step_by_step", "full_working"}:
+    if intent == "definition":
         if steps:
-            limit = 4 if verbosity < 0.75 else min(6, len(steps))
-            body = "\n".join(f"{i+1}. {s}" for i, s in enumerate(steps[:limit]))
-        else:
-            body = "1. Translate the wording into an equation.\n2. Isolate the variable carefully.\n3. Check the result."
-        if reveal_answer and internal:
-            body += f"\n\nSo the result is {internal}."
-        return body
+            return f"Here is the idea in context:\n- {steps[0]}"
+        return "This means identifying the mathematical idea being used and expressing it clearly."
 
-    if intent == "explain":
-        if steps:
-            body = "Here is the idea:\n" + "\n".join(f"- {s}" for s in steps[:3])
+    if intent in {"walkthrough", "step_by_step", "explain", "method", "concept"}:
+        if not steps:
+            if reveal_answer and internal:
+                return f"The result is {internal}."
+            return "I can explain the method, but I do not have enough structured steps yet."
+
+        if verbosity >= 0.66:
+            body = "\n".join(f"- {s}" for s in steps)
         else:
-            body = "Here is the idea:\n- Turn the wording into a clear mathematical relationship.\n- Solve it step by step."
+            body = "\n".join(f"- {s}" for s in steps[: min(3, len(steps))])
+
         if reveal_answer and internal:
-            body += f"\n\nThat leads to {internal}."
-        return body
+            return f"Walkthrough:\n{body}\n\nThat gives {internal}."
+        return f"Walkthrough:\n{body}"
 
+    # answer/default
     if reveal_answer and internal:
+        if result.answer_value and str(result.answer_value).startswith("x ="):
+            return f"The result is {result.answer_value}."
+        if result.answer_value:
+            return f"The answer is {result.answer_value}."
         return f"The result is {internal}."
 
     if steps:
-        return "\n".join(f"- {s}" for s in steps[:2])
-
-    return result.reply or "I can help solve this, but I need a little more structure from the question."
-
-
-class ConversationEngine:
-    def __init__(self, retriever: RetrievalEngine, generator: Optional[GeneratorEngine] = None):
-        self.retriever = retriever
-        self.generator = generator or GeneratorEngine()
-
-    def generate_response(
-        self,
-        raw_user_text: str,
-        tone: float,
-        verbosity: float,
-        transparency: float,
-        intent: str,
-        help_mode: str,
-        chat_history: Optional[List[Dict[str, Any]]] = None,
-        question_text: str = "",
-        options_text: str = "",
-        retrieval_context: str = "",
-    ) -> SolverResult:
-        user_text = (raw_user_text or "").strip()
-        question_text = (question_text or "").strip()
-        options_text = (options_text or "").strip()
-
-        question_block = "\n".join([x for x in [question_text, options_text] if x]).strip()
-        solver_input = user_text or question_block or question_text
-
-        quant_from_user = bool(user_text and is_quant_question(user_text))
-        quant_from_question = bool(question_text and is_quant_question(question_block or question_text))
-
-        if quant_from_user or quant_from_question:
-            solve_text = user_text if quant_from_user else (question_block or question_text)
-            result = solve_quant(solve_text)
-            result.help_mode = help_mode
-
-            reveal_answer = intent in {"answer", "full_working"} or transparency >= 0.85
-
-            chunks: List[RetrievedChunk] = []
-            if _should_retrieve(intent, result):
-                topic = result.topic or getattr(result, "detected_topic", None) or "general"
-                query = _retrieval_query(user_text, question_text, options_text)
-                chunks = self.retriever.search(query, topic=topic, intent=intent, k=3)
-
-            result.teaching_chunks = chunks
-            result.used_retrieval = bool(chunks)
-
-            core = _compose_quant_reply(
-                result,
-                intent=intent,
-                reveal_answer=reveal_answer,
-                verbosity=verbosity,
-            )
+        return steps[0]
 
-            if chunks and intent in {"hint", "method", "step_by_step", "full_working", "walkthrough", "explain"}:
-                core += "\n\nRelevant study notes:\n" + "\n".join(_teaching_lines(chunks))
+    return "I can help with this, but I cannot confidently solve it from the current parse alone yet."
 
-            result.reply = format_reply(core, tone, verbosity, transparency, help_mode)
-            return result
 
-        result = SolverResult(domain="general", solved=False, help_mode=help_mode)
+# -----------------------------
+# Intent / retrieval helpers
+# -----------------------------
 
-        prompt = (
-            "You are a helpful study assistant. Reply naturally and briefly. "
-            "Do not invent facts. If the user is asking for emotional support or general help, be supportive and practical.\n\n"
-            f"User message: {user_text}"
-        )
+def _normalize_text(text: str) -> str:
+    return re.sub(r"\s+", " ", (text or "").strip().lower())
+
+
+def _extract_keywords(text: str) -> Set[str]:
+    raw = re.findall(r"[a-zA-Z][a-zA-Z0-9_+-]*", text.lower())
+    stop = {
+        "the", "a", "an", "is", "are", "to", "of", "for", "and", "or", "in", "on",
+        "at", "by", "this", "that", "it", "be", "do", "i", "me", "my", "you",
+        "how", "what", "why", "give", "show", "please", "can"
+    }
+    return {w for w in raw if len(w) > 2 and w not in stop}
+
+
+def _infer_structure_terms(question_text: str, topic: Optional[str]) -> List[str]:
+    terms: List[str] = []
+    if topic and topic in STRUCTURE_KEYWORDS:
+        terms.extend(STRUCTURE_KEYWORDS[topic])
+
+    q = question_text.lower()
+
+    if "=" in q:
+        terms.extend(["equation", "solve"])
+    if "x" in q or "y" in q:
+        terms.extend(["variable", "isolate"])
+    if "/" in q or "divide" in q:
+        terms.extend(["divide", "undo operations"])
+    if "*" in q or "times" in q or "multiply" in q:
+        terms.extend(["multiply", "undo operations"])
+    if "%" in q or "percent" in q:
+        terms.extend(["percent", "percentage"])
+
+    return list(dict.fromkeys(terms))
+
+
+def _infer_mismatch_terms(topic: Optional[str], question_text: str) -> List[str]:
+    if not topic or topic not in MISMATCH_TERMS:
+        return []
+    q = question_text.lower()
+    terms = []
+    for term in MISMATCH_TERMS[topic]:
+        if term not in q:
+            terms.append(term)
+    return terms
+
+
+def _intent_keywords(intent: str) -> List[str]:
+    return INTENT_KEYWORDS.get(intent, [])
+
+
+def _is_direct_solve_request(text: str, intent: str) -> bool:
+    if intent == "answer":
+        return True
+
+    t = _normalize_text(text)
 
-        generated = self.generator.generate(prompt) if self.generator and self.generator.available() else None
-        if generated:
-            result.reply = format_reply(generated, tone, verbosity, transparency, help_mode)
-            result.used_generator = True
-            return result
+    if any(re.search(p, t) for p in DIRECT_SOLVE_PATTERNS):
+        if not any(word in t for word in ["how", "explain", "why", "method", "hint", "define", "definition", "step"]):
+            return True
+
+    return False
+
+
+def should_retrieve(intent: str, solved: bool, raw_user_text: str) -> bool:
+    if intent in RETRIEVAL_ALLOWED_INTENTS:
+        return True
+
+    if not solved:
+        return True
+
+    if _is_direct_solve_request(raw_user_text, intent):
+        return False
+
+    return False
+
+
+def _score_chunk(
+    chunk: RetrievedChunk,
+    intent: str,
+    topic: Optional[str],
+    question_text: str,
+) -> float:
+    text = f"{getattr(chunk, 'topic', '')} {chunk.text}".lower()
+    score = 0.0
+
+    # topic match
+    if topic:
+        chunk_topic = (getattr(chunk, "topic", "") or "").lower()
+        if chunk_topic == topic.lower():
+            score += 4.0
+        elif topic.lower() in text:
+            score += 2.0
+
+    # structure match
+    structure_terms = _infer_structure_terms(question_text, topic)
+    for term in structure_terms:
+        if term.lower() in text:
+            score += 1.5
+
+    # intent match
+    for term in _intent_keywords(intent):
+        if term.lower() in text:
+            score += 1.2
+
+    # question keyword overlap
+    q_keywords = _extract_keywords(question_text)
+    overlap = sum(1 for kw in q_keywords if kw in text)
+    score += min(overlap * 0.4, 3.0)
+
+    # penalties for obvious mismatch
+    mismatch_terms = _infer_mismatch_terms(topic, question_text)
+    for bad in mismatch_terms:
+        if bad.lower() in text:
+            score -= 2.5
+
+    return score
+
+
+def _filter_retrieved_chunks(
+    chunks: List[RetrievedChunk],
+    intent: str,
+    topic: Optional[str],
+    question_text: str,
+    min_score: float = 2.5,
+    max_chunks: int = 3,
+) -> List[RetrievedChunk]:
+    scored = []
+    for chunk in chunks:
+        s = _score_chunk(chunk, intent, topic, question_text)
+        if s >= min_score:
+            scored.append((s, chunk))
+
+    scored.sort(key=lambda x: x[0], reverse=True)
+    return [chunk for _, chunk in scored[:max_chunks]]
+
+
+def _build_retrieval_query(
+    raw_user_text: str,
+    question_text: str,
+    intent: str,
+    topic: Optional[str],
+    solved: bool,
+) -> str:
+    parts: List[str] = []
+
+    base = question_text.strip() if question_text.strip() else raw_user_text.strip()
+    if base:
+        parts.append(base)
+
+    if topic:
+        parts.append(topic)
+
+    if intent in {"definition", "concept"}:
+        parts.append("definition concept explanation")
+    elif intent in {"walkthrough", "step_by_step", "method", "instruction"}:
+        parts.append("method steps worked example")
+    elif intent == "hint":
+        parts.append("hint strategy first step")
+    elif intent == "explain":
+        parts.append("explanation reasoning")
+    elif not solved:
+        parts.append("teaching explanation method")
+
+    return " ".join(parts).strip()
+
+
+# -----------------------------
+# Public entry point
+# -----------------------------
+
+def generate_response(
+    raw_user_text: str,
+    tone: float = 0.5,
+    verbosity: float = 0.5,
+    transparency: float = 0.5,
+    retrieval_engine: Optional[RetrievalEngine] = None,
+    generator_engine: Optional[GeneratorEngine] = None,
+    retrieval_context: Optional[List[RetrievedChunk]] = None,
+    chat_history: Optional[List[Dict[str, Any]]] = None,
+    question_text: Optional[str] = None,
+) -> Dict[str, Any]:
+    solver_input = (question_text or raw_user_text or "").strip()
+    user_text = (raw_user_text or "").strip()
+
+    intent = detect_intent(user_text)
+    help_mode = intent_to_help_mode(intent)
+
+    reveal_answer = help_mode == "answer" or transparency >= 0.8
+
+    result = SolverResult(
+        domain="general",
+        solved=False,
+        answer_letter=None,
+        answer_value=None,
+        internal_answer=None,
+        steps=[],
+        topic=None,
+    )
+
+    used_retrieval = False
+    used_generator = False
+    selected_chunks: List[RetrievedChunk] = []
+
+    if is_quant_question(solver_input):
+        result = solve_quant(solver_input)
+
+    reply = _compose_quant_reply(
+        result=result,
+        intent=intent,
+        reveal_answer=reveal_answer,
+        verbosity=verbosity,
+    )
+
+    allow_retrieval = should_retrieve(
+        intent=intent,
+        solved=bool(result.solved),
+        raw_user_text=user_text or solver_input,
+    )
+
+    # Use passed-in retrieval context only if retrieval is allowed
+    if allow_retrieval and retrieval_context:
+        filtered = _filter_retrieved_chunks(
+            chunks=retrieval_context,
+            intent=intent,
+            topic=result.topic,
+            question_text=solver_input,
+        )
+        if filtered:
+            selected_chunks = filtered
+            used_retrieval = True
+
+    # Otherwise retrieve fresh if allowed
+    elif allow_retrieval and retrieval_engine is not None:
+        query = _build_retrieval_query(
+            raw_user_text=user_text,
+            question_text=solver_input,
+            intent=intent,
+            topic=result.topic,
+            solved=bool(result.solved),
+        )
+        retrieved = retrieval_engine.search(query, top_k=6)
+        filtered = _filter_retrieved_chunks(
+            chunks=retrieved,
+            intent=intent,
+            topic=result.topic,
+            question_text=solver_input,
+        )
+        if filtered:
+            selected_chunks = filtered
+            used_retrieval = True
+
+    # Add teaching notes only if they survived filtering
+    if selected_chunks:
+        reply = f"{reply}\n\nRelevant study notes:\n" + "\n".join(_teaching_lines(selected_chunks))
+
+    # Optional generator fallback for non-quant / weak cases
+    if not result.solved and generator_engine is not None:
+        try:
+            generated = generator_engine.generate(
+                user_text=user_text or solver_input,
+                intent=intent,
+                topic=result.topic,
+                chat_history=chat_history or [],
+            )
+            if generated and generated.strip():
+                reply = generated.strip()
+                used_generator = True
+        except Exception:
+            pass
+
+    reply = format_reply(
+        text=reply,
+        tone=tone,
+        verbosity=verbosity,
+        transparency=transparency,
+    )
 
-        fallback = "I can help with the current question, explain a method, or talk through the next step."
-        result.reply = format_reply(fallback, tone, verbosity, transparency, help_mode)
-        return result
+    return {
+        "reply": short_lines(reply),
+        "meta": {
+            "domain": result.domain,
+            "solved": result.solved,
+            "help_mode": help_mode,
+            "answer_letter": result.answer_letter,
+            "answer_value": result.answer_value,
+            "topic": result.topic,
+            "used_retrieval": used_retrieval,
+            "used_generator": used_generator,
+        },
+    }