feat(task): :sparkles: analyze oj wrong answer (#10)

Accept a coding problem and incorrect answer as input. Start a conversation to help student figure out whats wrong

edu_assistant/learning_tasks/__init__.py

@@ -1,3 +1,4 @@
+from .coding_problem import CodingProblemAnalysis
 from .qa import QaTask
 
-__all__ = ["QaTask"]
+__all__ = ["QaTask", "CodingProblemAnalysis"]

edu_assistant/learning_tasks/base.py

@@ -1,2 +1,6 @@
+import uuid
+
+
 class BaseTask:
-    pass
+    def _create_session_id(self) -> str:
+        return str(uuid.uuid1())

edu_assistant/learning_tasks/coding_problem.py

@@ -0,0 +1,160 @@
+from langchain import ConversationChain, PromptTemplate
+from langchain.chains import ConversationalRetrievalChain
+from langchain.chains.base import Chain
+from langchain.memory import ConversationBufferMemory
+from langchain.schema import BaseRetriever
+from pydantic import BaseModel, Field
+
+from edu_assistant.learning_tasks.base import BaseTask
+from edu_assistant.utils.langchain_utils import load_llm
+
+TEMPLATE = """The following is a friendly conversation between a human and an ai.
+The ai is talkative and provides lots of specific details from its context.
+If the ai does not know the answer to a question, it truthfully says it does not know.
+The ai act following below instructions:
+---
+{instruction}
+---
+
+The coding problem:
+---
+{problem}
+---
+
+Student's code:
+---
+{answer}
+---
+
+Current conversation:
+{{history}}
+Human: {{input}}
+AI:"""
+
+DEFAULT_INSTRUCTION = """Act as a c++ professional to check student's code.
+The code is written by a student aged 5-10 and mostly like to buggy or bad performanced.
+"""
+
+DEFAULT_FIRST_QUESTION = "请问这段代码中有什么问题吗？"
+
+
+class CodingProblem(BaseModel):
+    question: str = Field()
+    standard_answer: str = Field(default="")
+    analysis: str = Field(default="")
+    extra: list[str] = Field(default_factory=lambda _: list())
+
+    # TODO: Add cache to expr function with pydantic 2 computed_field decorator.
+    # Wait for langchain to support pydantic2.
+
+    def expr(self, lang=""):
+        expr = f"Question:\n```\n{self.question}\n```\n"
+        expr += (
+            f"Question Standard Answer (There might be others):\n```{lang}\n{self.standard_answer}```"
+            if self.standard_answer
+            else ""
+        )
+        expr += f"Solution Analysis:\n```{self.analysis}```\n" if self.analysis else ""
+        expr += "".join(self.extra) + "\n"
+        return expr
+
+    def __str__(self):
+        return self.expr()
+
+
+class CodingAnswer(BaseModel):
+    answer: str = Field()
+    extra: list[str] = Field(default="")
+
+    def expr(self, lang=""):
+        expr = f"Answer:\n```{lang}\n{self.answer}\n```\n"
+        expr += "".join(self.extra) + "\n"
+        return expr
+
+    def __str__(self):
+        return self.expr()
+
+
+class CodingProblemAnalysis(BaseTask):
+    def __init__(self, instruction: str = DEFAULT_INSTRUCTION, lang: str = "", knowledge: BaseRetriever = None):
+        assert lang in ["python", "cpp", "java", "javascript", "go", "c#", ""]
+
+        self.lang = lang
+        self.instruction = instruction
+        self._session_store = {}
+        self._knowledge = knowledge
+
+    def start_analysis(self, problem: CodingProblem, answer: CodingAnswer, first_question: str = None) -> dict:
+        """start analysis of a coding problem and incorrect answer.
+
+        Args:
+            problem (CodingProblem): a coding problem
+            answer (CodingAnswer): a coding problem answer
+
+        Returns:
+            dict: question answer and metadata
+        """
+        chain = self._build_chain(problem, answer)
+        session_id = self._create_session_id()
+        self._session_store[session_id] = chain
+
+        result = chain({"input": first_question if first_question else DEFAULT_FIRST_QUESTION, "history": ""})
+
+        result["session_id"] = session_id
+
+        return result
+
+    def ask(self, question: str, session_id: str) -> dict:
+        """further ask question on a coding problem.
+
+        Args:
+            question (str): question to llm.
+            session_id (str): specify a problem and answer session.
+
+        Returns:
+            dict: question answer and metadata
+        """
+        assert question
+
+        chain = self._session_store[session_id]
+
+        result = chain({"input": question})
+
+        result["session_id"] = session_id
+
+        return result
+
+    def _build_chain(self, problem: CodingProblem, answer: CodingAnswer) -> Chain:
+        llm = load_llm()
+        memory = ConversationBufferMemory()
+        prompt = PromptTemplate.from_template(
+            TEMPLATE.format(
+                instruction=self.instruction, problem=problem.expr(lang=self.lang), answer=answer.expr(lang=self.lang)
+            )
+        )
+
+        if not self._knowledge:
+            return ConversationChain(
+                llm=llm,
+                memory=memory,
+                prompt=prompt,
+            )
+        else:
+            return ConversationalRetrievalChain.from_llm(
+                llm=llm,
+                memory=memory,
+                retriever=self._knowledge,
+                condense_question_llm=llm,
+                return_source_documents=True,
+                combine_docs_chain_kwargs={"prompt": prompt},
+            )
+
+    @staticmethod
+    def build_coding_problem(question: str, standard_answer: str = "", analysis: str = "", extra: list[str] = None):
+        extra = [] if extra is None else extra
+        return CodingProblem(question=question, standard_answer=standard_answer, analysis=analysis, extra=extra)
+
+    @staticmethod
+    def build_coding_answer(answer: str, extra: list[str] = None):
+        extra = [] if extra is None else extra
+        return CodingAnswer(answer=answer, extra=extra)

edu_assistant/learning_tasks/qa.py

@@ -1,36 +1,47 @@
-import random
-import sys
-
-from langchain import PromptTemplate
-from langchain.chains import ConversationalRetrievalChain, ConversationChain
+from langchain import LLMChain, PromptTemplate
+from langchain.chains import (
+    ConversationalRetrievalChain,
+    ConversationChain,
+    RetrievalQA,
+)
 from langchain.chains.base import Chain
 from langchain.memory import ConversationBufferMemory
 from langchain.schema import BaseRetriever
 
 from edu_assistant.learning_tasks.base import BaseTask
-from edu_assistant.utils.langchain_utils import load_llm, update_chat_memory
+from edu_assistant.utils.langchain_utils import load_llm
 
-TEMPLATE = """{instruction}
-The following is a friendly conversation between a human and an AI.
-The AI is talkative and provides lots of specific details from its context.
-If the AI does not know the answer to a question, it truthfully says it does not know.
+TEMPLATE_CHAT = """The following is a friendly conversation between a human and an ai.
+The ai is talkative and provides lots of specific details from its context.
+If the ai does not know the answer to a question, it truthfully says it does not know.
+The ai act following below instructions:
+---
+{instruction}
+---
 
 Current conversation:
 {{history}}
 Human: {{input}}
 AI:"""
 
+TEMPLATE_ONCE = """The following is a friendly conversation between a human and an ai.
+The ai is talkative and provides lots of specific details from its context.
+If the ai does not know the answer to a question, it truthfully says it does not know.
+The ai act following below instructions:
+---
+{instruction}
+---
+
+{{input}}
+"""
+
 
 class QaTask(BaseTask):
-    _qa: Chain
     _session_store: dict
+    _knowledge: BaseRetriever | None
+    _qa_once: Chain
 
-    def __init__(
-        self,
-        instruction: str = "",
-        knowledge: BaseRetriever = None,
-        session_store: dict[int, ConversationBufferMemory] = None,
-    ):
+    def __init__(self, instruction: str = "", knowledge: BaseRetriever = None):
         """Create a new QaTask service.
 
         Args:
@@ -38,36 +49,66 @@ class QaTask(BaseTask):
             knowledge (BaseRetriever, optional): Answer question with this knowledge retriever.
                 If not set, will not use knowledge to answer question.
                 Defaults to None.
-            session_store (dict, optional): External chat history store. Defaults to None.
+            session_store (dict, optional): chat history store. Defaults to None.
                 If not set, will use internal memory to store chat history. Which will be lost after restart and might
                 cost huge memory.
         """
-        self._prompt = TEMPLATE.format(instruction=instruction)
-        self._qa = self._build_chain(knowledge)
-        self._session_store = {} if not session_store else session_store
+        self._chat_prompt = PromptTemplate.from_template(TEMPLATE_CHAT.format(instruction=instruction))
+        self._once_prompt = PromptTemplate.from_template(TEMPLATE_ONCE.format(instruction=instruction))
 
-    def _build_chain(self, knowledge):
-        if not knowledge:
-            return ConversationChain(llm=load_llm(), prompt=PromptTemplate.from_template(self._prompt))
+        self._session_store = {}
+        self._knowledge = knowledge
+
+        self._qa_once = self._build_once_chain()
+
+    def _build_once_chain(self):
+        if not self._knowledge:
+            return LLMChain(
+                llm=load_llm(),
+                prompt=self._once_prompt,
+                memory=ConversationBufferMemory(),
+            )
+        else:
+            return RetrievalQA.from_llm(
+                llm=load_llm(),
+                retriever=self._knowledge,
+                return_source_documents=True,
+                prompt=self._once_prompt,
+            )
+
+    def _build_chat_chain(self):
+        if not self._knowledge:
+            return ConversationChain(
+                llm=load_llm(),
+                memory=ConversationBufferMemory(),
+                prompt=self._chat_prompt,
+            )
         else:
             return ConversationalRetrievalChain.from_llm(
                 llm=load_llm(),
-                retriever=knowledge,
+                retriever=self._knowledge,
                 condense_question_llm=load_llm(),
                 return_source_documents=True,
-                combine_docs_chain_kwargs={"prompt": PromptTemplate.from_template(self._prompt)},
+                combine_docs_chain_kwargs={"prompt": PromptTemplate.from_template(self._chat_prompt)},
+                memory=ConversationBufferMemory(),
             )
 
-    def ask(self, question: str, session: bool = True, session_id: int = None, session_mem: list | None = None) -> dict:
+    def ask(
+        self,
+        question: str,
+        session: bool = True,
+        session_id: str = None,
+        session_mem: ConversationBufferMemory | None = None,
+    ) -> dict:
         """ask a question with chat history.
 
         Args:
             question (str): question to llm.
             session (bool, optional): whether use and store chat history. Defaults to False.
                 if session_id is not set, a new session will be created.
-            session_id (int, optional): specify a history session. Defaults to None.
-            session_mem (list | None, optional): specify chat history. Defaults to None.
-                if session_id is also set, chat history will be fully replaced.
+            session_id (str, optional): specify a history qa session. Defaults to None.
+            session_mem (list | None, optional): specify session memory. Defaults to None.
+                if session_id is also set, memory will be replaced by passed one.
 
         Returns:
             dict: question answer and metadata.
@@ -76,41 +117,27 @@ class QaTask(BaseTask):
         """
 
         if session:
-            session_id = session_id or self._create_session_id()
-            memory = session_mem or self._get_session_mem(session_id)
+            args = {"input": question}
+            if session_id and session_id in self._session_store:
+                chain = self._session_store[session_id]
+            else:
+                session_id = self._create_session_id()
+                chain = self._create_session_chain(session_id)
+        else:
+            args = {"input": question, "history": ""}
+            chain = self._qa_once
 
-            result = self._ask(question, memory)
+        if session_mem:
+            chain.memory = session_mem
 
-            update_chat_memory(memory, question, result["response"])
-            self._update_session_mem(session_id, memory)
-        else:
-            result = self._ask(question)
+        result = chain(args)
 
         if session_id:
             result["session_id"] = session_id
 
         return result
 
-    def _ask(self, question, memory: ConversationBufferMemory = None) -> dict:
-        if memory is None:
-            return self._qa({"input": question, "history": ""})
-        else:
-            return self._qa({"input": question, "history": memory.chat_memory})
-
-    def _get_session_mem(self, session_id):
-        if session_id not in self._session_store:
-            memory = self._init_memory(session_id)
-        else:
-            memory = self._session_store.get(session_id)
-
-        return memory
-
-    def _update_session_mem(self, session_id, memory):
-        self._session_store[session_id] = memory
-
-    def _create_session_id(self):
-        return random.randint(1, sys.maxsize)
-
-    def _init_memory(self, session_id) -> ConversationBufferMemory:
-        # TODO: redis memory store
-        return ConversationBufferMemory(memory_key="chat_history", return_messages=True, output_key="answer")
+    def _create_session_chain(self, session_id) -> ConversationChain:
+        chain = self._build_chat_chain()
+        self._session_store[session_id] = chain
+        return chain

edu_assistant/utils/langchain_utils.py

@@ -1,10 +1,12 @@
 import os
+from functools import lru_cache
 
 from langchain.chat_models import AzureChatOpenAI, ChatOpenAI
 from langchain.chat_models.base import BaseChatModel
 from langchain.memory.chat_memory import BaseChatMemory
 
 
+@lru_cache(maxsize=1)
 def load_llm() -> BaseChatModel:
     if os.environ.get("AZURE_OPENAI"):
         llm = AzureChatOpenAI(

examples/coding_problem.py

@@ -0,0 +1,23 @@
+import json
+
+from edu_assistant.learning_tasks import CodingProblemAnalysis
+
+# 创建一个CodingProblem实例
+problem = CodingProblemAnalysis.build_coding_problem(
+    question="请编写一个函数，该函数接收一个整数列表，并返回该列表的最大值。",
+    standard_answer="def find_max(lst):\n\treturn max(lst)",
+    analysis="在这个问题中，我们需要使用Python的内置函数max来找到列表的最大值。",
+)
+
+# 创建一个CodingAnswer实例
+answer = CodingProblemAnalysis.build_coding_answer(
+    answer="def find_max(lst):\n\treturn lst[0]",
+)
+
+analysis = CodingProblemAnalysis(lang="python")
+result = analysis.start_analysis(problem, answer)
+print(json.dumps(result, ensure_ascii=False, indent=4))
+
+next_question = "如果改成返回最小值呢？"
+result = analysis.ask(next_question, result["session_id"])
+print(json.dumps(result, ensure_ascii=False, indent=4))

examples/qa.py

@@ -19,8 +19,8 @@ def qa_once():
 
 def qa_twice():
     task = QaTask()
-    task.ask("请问如何释放一个数组？")
-    result = task.ask("那指针呢？")
+    result = task.ask("请问如何释放一个数组？")
+    result = task.ask("那指针呢？", session_id=result["session_id"])
     print(json.dumps(result, ensure_ascii=False, indent=4))
 
 

poetry.lock

@@ -1136,19 +1136,20 @@ reference = "aliyun"
 
 [[package]]
 name = "langchain"
-version = "0.0.179"
+version = "0.0.234"
 description = "Building applications with LLMs through composability"
 optional = false
 python-versions = ">=3.8.1,<4.0"
 files = [
-    {file = "langchain-0.0.179-py3-none-any.whl", hash = "sha256:1af609e32d9297413ca7162efa97f001c8d87217533d3b5ca7cb87b44ddddf35"},
-    {file = "langchain-0.0.179.tar.gz", hash = "sha256:9c1ddddf4b24f5c0f981625dd248cf61bcfda7c1ebecf7d780fedd9b36bbc5ae"},
+    {file = "langchain-0.0.234-py3-none-any.whl", hash = "sha256:a287f0b944fb1b48cc107cedb8c1ad052e0559327c7658ae20e6ce2e8e851122"},
+    {file = "langchain-0.0.234.tar.gz", hash = "sha256:fdb5ba8176497e5bdd7cbb7594125b1149d306f8c9ed31750160271fece356ee"},
 ]
 
 [package.dependencies]
 aiohttp = ">=3.8.3,<4.0.0"
 async-timeout = {version = ">=4.0.0,<5.0.0", markers = "python_version < \"3.11\""}
 dataclasses-json = ">=0.5.7,<0.6.0"
+langsmith = ">=0.0.5,<0.0.6"
 numexpr = ">=2.8.4,<3.0.0"
 numpy = ">=1,<2"
 openapi-schema-pydantic = ">=1.2,<2.0"
@@ -1159,15 +1160,17 @@ SQLAlchemy = ">=1.4,<3"
 tenacity = ">=8.1.0,<9.0.0"
 
 [package.extras]
-all = ["O365 (>=2.0.26,<3.0.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.2.6,<0.3.0)", "arxiv (>=1.4,<2.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "azure-ai-formrecognizer (>=3.2.1,<4.0.0)", "azure-ai-vision (>=0.11.1b1,<0.12.0)", "azure-cognitiveservices-speech (>=1.28.0,<2.0.0)", "azure-cosmos (>=4.4.0b1,<5.0.0)", "azure-identity (>=1.12.0,<2.0.0)", "beautifulsoup4 (>=4,<5)", "clickhouse-connect (>=0.5.14,<0.6.0)", "cohere (>=3,<4)", "deeplake (>=3.3.0,<4.0.0)", "docarray[hnswlib] (>=0.32.0,<0.33.0)", "duckduckgo-search (>=2.8.6,<3.0.0)", "elasticsearch (>=8,<9)", "faiss-cpu (>=1,<2)", "google-api-python-client (==2.70.0)", "google-search-results (>=2,<3)", "gptcache (>=0.1.7)", "html2text (>=2020.1.16,<2021.0.0)", "huggingface_hub (>=0,<1)", "jina (>=3.14,<4.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lancedb (>=0.1,<0.2)", "langkit (>=0.0.1.dev3,<0.1.0)", "lark (>=1.1.5,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "manifest-ml (>=0.0.1,<0.0.2)", "neo4j (>=5.8.1,<6.0.0)", "networkx (>=2.6.3,<3.0.0)", "nlpcloud (>=1,<2)", "nltk (>=3,<4)", "nomic (>=1.0.43,<2.0.0)", "openai (>=0,<1)", "openlm (>=0.0.5,<0.0.6)", "opensearch-py (>=2.0.0,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pexpect (>=4.8.0,<5.0.0)", "pgvector (>=0.1.6,<0.2.0)", "pinecone-client (>=2,<3)", "pinecone-text (>=0.4.2,<0.5.0)", "psycopg2-binary (>=2.9.5,<3.0.0)", "pyowm (>=3.3.0,<4.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pytesseract (>=0.3.10,<0.4.0)", "pyvespa (>=0.33.0,<0.34.0)", "qdrant-client (>=1.1.2,<2.0.0)", "redis (>=4,<5)", "requests-toolbelt (>=1.0.0,<2.0.0)", "sentence-transformers (>=2,<3)", "spacy (>=3,<4)", "steamship (>=2.16.9,<3.0.0)", "tensorflow-text (>=2.11.0,<3.0.0)", "tiktoken (>=0.3.2,<0.4.0)", "torch (>=1,<3)", "transformers (>=4,<5)", "weaviate-client (>=3,<4)", "wikipedia (>=1,<2)", "wolframalpha (==5.0.0)"]
-azure = ["azure-ai-formrecognizer (>=3.2.1,<4.0.0)", "azure-ai-vision (>=0.11.1b1,<0.12.0)", "azure-cognitiveservices-speech (>=1.28.0,<2.0.0)", "azure-core (>=1.26.4,<2.0.0)", "azure-cosmos (>=4.4.0b1,<5.0.0)", "azure-identity (>=1.12.0,<2.0.0)", "openai (>=0,<1)"]
+all = ["O365 (>=2.0.26,<3.0.0)", "aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.3,<0.4)", "arxiv (>=1.4,<2.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "awadb (>=0.3.3,<0.4.0)", "azure-ai-formrecognizer (>=3.2.1,<4.0.0)", "azure-ai-vision (>=0.11.1b1,<0.12.0)", "azure-cognitiveservices-speech (>=1.28.0,<2.0.0)", "azure-cosmos (>=4.4.0b1,<5.0.0)", "azure-identity (>=1.12.0,<2.0.0)", "beautifulsoup4 (>=4,<5)", "clarifai (>=9.1.0)", "clickhouse-connect (>=0.5.14,<0.6.0)", "cohere (>=3,<4)", "deeplake (>=3.6.8,<4.0.0)", "docarray[hnswlib] (>=0.32.0,<0.33.0)", "duckduckgo-search (>=3.8.3,<4.0.0)", "elasticsearch (>=8,<9)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "google-api-python-client (==2.70.0)", "google-auth (>=2.18.1,<3.0.0)", "google-search-results (>=2,<3)", "gptcache (>=0.1.7)", "html2text (>=2020.1.16,<2021.0.0)", "huggingface_hub (>=0,<1)", "jina (>=3.14,<4.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lancedb (>=0.1,<0.2)", "langkit (>=0.0.6,<0.1.0)", "lark (>=1.1.5,<2.0.0)", "libdeeplake (>=0.0.60,<0.0.61)", "lxml (>=4.9.2,<5.0.0)", "manifest-ml (>=0.0.1,<0.0.2)", "marqo (>=0.11.0,<0.12.0)", "momento (>=1.5.0,<2.0.0)", "nebula3-python (>=3.4.0,<4.0.0)", "neo4j (>=5.8.1,<6.0.0)", "networkx (>=2.6.3,<3.0.0)", "nlpcloud (>=1,<2)", "nltk (>=3,<4)", "nomic (>=1.0.43,<2.0.0)", "octoai-sdk (>=0.1.1,<0.2.0)", "openai (>=0,<1)", "openlm (>=0.0.5,<0.0.6)", "opensearch-py (>=2.0.0,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pexpect (>=4.8.0,<5.0.0)", "pgvector (>=0.1.6,<0.2.0)", "pinecone-client (>=2,<3)", "pinecone-text (>=0.4.2,<0.5.0)", "psycopg2-binary (>=2.9.5,<3.0.0)", "pymongo (>=4.3.3,<5.0.0)", "pyowm (>=3.3.0,<4.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pytesseract (>=0.3.10,<0.4.0)", "pyvespa (>=0.33.0,<0.34.0)", "qdrant-client (>=1.3.1,<2.0.0)", "rdflib (>=6.3.2,<7.0.0)", "redis (>=4,<5)", "requests-toolbelt (>=1.0.0,<2.0.0)", "sentence-transformers (>=2,<3)", "singlestoredb (>=0.7.1,<0.8.0)", "spacy (>=3,<4)", "steamship (>=2.16.9,<3.0.0)", "tensorflow-text (>=2.11.0,<3.0.0)", "tigrisdb (>=1.0.0b6,<2.0.0)", "tiktoken (>=0.3.2,<0.4.0)", "torch (>=1,<3)", "transformers (>=4,<5)", "weaviate-client (>=3,<4)", "wikipedia (>=1,<2)", "wolframalpha (==5.0.0)"]
+azure = ["azure-ai-formrecognizer (>=3.2.1,<4.0.0)", "azure-ai-vision (>=0.11.1b1,<0.12.0)", "azure-cognitiveservices-speech (>=1.28.0,<2.0.0)", "azure-core (>=1.26.4,<2.0.0)", "azure-cosmos (>=4.4.0b1,<5.0.0)", "azure-identity (>=1.12.0,<2.0.0)", "azure-search-documents (==11.4.0a20230509004)", "openai (>=0,<1)"]
+clarifai = ["clarifai (>=9.1.0)"]
 cohere = ["cohere (>=3,<4)"]
 docarray = ["docarray[hnswlib] (>=0.32.0,<0.33.0)"]
 embeddings = ["sentence-transformers (>=2,<3)"]
-extended-testing = ["atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "chardet (>=5.1.0,<6.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "psychicapi (>=0.2,<0.3)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "telethon (>=1.28.5,<2.0.0)", "tqdm (>=4.48.0)", "zep-python (>=0.25,<0.26)"]
-llms = ["anthropic (>=0.2.6,<0.3.0)", "cohere (>=3,<4)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (>=0,<1)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)"]
+extended-testing = ["atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.0.7,<0.0.8)", "chardet (>=5.1.0,<6.0.0)", "esprima (>=4.0.1,<5.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "openai (>=0,<1)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "tqdm (>=4.48.0)", "zep-python (>=0.32)"]
+javascript = ["esprima (>=4.0.1,<5.0.0)"]
+llms = ["anthropic (>=0.3,<0.4)", "clarifai (>=9.1.0)", "cohere (>=3,<4)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (>=0,<1)", "openllm (>=0.1.19)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)"]
 openai = ["openai (>=0,<1)", "tiktoken (>=0.3.2,<0.4.0)"]
-qdrant = ["qdrant-client (>=1.1.2,<2.0.0)"]
+qdrant = ["qdrant-client (>=1.3.1,<2.0.0)"]
 text-helpers = ["chardet (>=5.1.0,<6.0.0)"]
 
 [package.source]
@@ -1175,6 +1178,26 @@ type = "legacy"
 url = "http://mirrors.aliyun.com/pypi/simple"
 reference = "aliyun"
 
+[[package]]
+name = "langsmith"
+version = "0.0.5"
+description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
+optional = false
+python-versions = ">=3.8.1,<4.0"
+files = [
+    {file = "langsmith-0.0.5-py3-none-any.whl", hash = "sha256:c9ce19cf7a45d4b9ef74b3133ace4d0583bc992383296d03c05065e8f871e01f"},
+    {file = "langsmith-0.0.5.tar.gz", hash = "sha256:ffad2fc638cfee8c9d27c9eae2fa3c3f9ec423bf443b1dc44cc8184fa34cd6b2"},
+]
+
+[package.dependencies]
+pydantic = ">=1,<2"
+requests = ">=2,<3"
+
+[package.source]
+type = "legacy"
+url = "http://mirrors.aliyun.com/pypi/simple"
+reference = "aliyun"
+
 [[package]]
 name = "marshmallow"
 version = "3.19.0"
@@ -2517,4 +2540,4 @@ reference = "aliyun"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "bf7411574c4cfde57b9017ce88a88019ca51f05c24b086bb4477e2ff85201b51"
+content-hash = "8af40fe57d5e45ae818625a895be0c24a0dce9023aaf7bd9c05471a194e7a32f"

pyproject.toml

@@ -10,7 +10,7 @@ packages = [{include = "edu_assistant"}]
 [tool.poetry.dependencies]
 python = "^3.10"
 fastapi = "^0.95.1"
-langchain = "^0.0.179"
+langchain = "^0.0.234"
 openai = "^0.27.4"
 pydantic = "^1.10.7"
 tenacity = "^8.2.2"

tests/learning_tasks/test_base_task.py

@@ -0,0 +1,11 @@
+from edu_assistant.learning_tasks.base import BaseTask
+
+
+def test_create_session_id():
+    task = BaseTask()
+    id1 = task._create_session_id()
+    id2 = task._create_session_id()
+
+    assert isinstance(id1, str)
+    assert isinstance(id2, str)
+    assert id1 != id2

tests/learning_tasks/test_coding_problem.py

@@ -0,0 +1,39 @@
+from unittest import TestCase
+from unittest.mock import MagicMock, patch
+
+from edu_assistant.learning_tasks import CodingProblemAnalysis
+
+
+class TestCodingProblemAnalysis(TestCase):
+    def setUp(self):
+        self.analysis = CodingProblemAnalysis()
+
+    @patch.object(
+        CodingProblemAnalysis,
+        "_build_chain",
+        return_value=MagicMock(return_value={"response": "Expected Result"}),
+    )
+    def test_start_analysis(self, mock_chain):
+        problem = MagicMock()
+        answer = MagicMock()
+        result = self.analysis.start_analysis(problem, answer, "First question")
+
+        self.assertIn("session_id", result)
+        self.assertIn("response", result)
+        self.assertEqual(result["response"], "Expected Result")
+        mock_chain.assert_called_once_with(problem, answer)
+
+    @patch.object(
+        CodingProblemAnalysis,
+        "_build_chain",
+        return_value=MagicMock(return_value={"response": "Expected Result"}),
+    )
+    def test_ask(self, mock_chain):
+        session_id = self.analysis.start_analysis(MagicMock(), MagicMock(), "First question")["session_id"]
+        result = self.analysis.ask("New question", session_id)
+        self.assertIn("session_id", result)
+        self.assertIn("response", result)
+        self.assertEqual(result["response"], "Expected Result")
+        self.assertEqual(result["session_id"], session_id)
+        self.analysis._session_store[session_id].has_calls(2)
+        self.analysis._session_store[session_id].assert_called_with({"input": "New question"})

tests/learning_tasks/test_qa.py

@@ -0,0 +1,50 @@
+from unittest.mock import MagicMock, patch
+
+from langchain import PromptTemplate
+
+from edu_assistant.learning_tasks import QaTask
+from edu_assistant.learning_tasks.qa import TEMPLATE_CHAT, TEMPLATE_ONCE
+
+
+@patch.object(QaTask, "_build_once_chain")
+def test_init_without_knowledge(mocked_build_once_chain):
+    task = QaTask(instruction="test")
+
+    assert task._chat_prompt == PromptTemplate.from_template(TEMPLATE_CHAT.format(instruction="test"))
+    assert task._once_prompt == PromptTemplate.from_template(TEMPLATE_ONCE.format(instruction="test"))
+    assert task._knowledge is None
+    mocked_build_once_chain.assert_called_once()
+
+
+@patch.object(QaTask, "_build_once_chain")
+@patch.object(QaTask, "_create_session_chain")
+def test_ask_with_session(mocked_create_session_chain, mocked_build_once_chain):
+    mocked_chain = MagicMock(return_value={"response": "ok"})
+    mocked_build_once_chain.return_value = mocked_chain
+    mocked_create_session_chain.return_value = mocked_chain
+
+    task = QaTask(instruction="test")
+
+    with patch.object(task, "_create_session_id") as mock_create_id:
+        mock_create_id.return_value = 123
+        result = task.ask("how are you?", session=True)
+
+    mock_create_id.assert_called_once()
+    mocked_create_session_chain.assert_called_once_with(123)
+    assert "session_id" in result
+    assert result["session_id"] == 123
+    assert "response" in result
+    assert result["response"] == "ok"
+
+
+@patch.object(QaTask, "_build_once_chain")
+def test_ask_without_session(mocked_build_once_chain):
+    mocked_llm = MagicMock()
+    mocked_llm.run.return_value = {"result": "ok"}
+    mocked_build_once_chain.return_value = mocked_llm
+    task = QaTask(instruction="test")
+
+    result = task.ask("how are you?", session=False)
+
+    mocked_build_once_chain.assert_called_once()
+    assert "session_id" not in result

tests/unit_tests/learning_tasks/test_qa.py

@@ -1,53 +0,0 @@
-import unittest
-from unittest.mock import MagicMock, patch
-
-from langchain.memory import ConversationBufferMemory
-
-from edu_assistant.learning_tasks import QaTask
-
-
-class TestQaTask(unittest.TestCase):
-    @patch.object(QaTask, "_build_chain")
-    def test_init(self, mock_build_chain):
-        mock_build_chain.return_value = MagicMock()
-        QaTask()
-        mock_build_chain.assert_called_once()
-
-    @patch.object(QaTask, "_build_chain")
-    def test_ask_with_session(self, mock_build_chain):
-        mock_build_chain.return_value = MagicMock()
-        qa_task = QaTask()
-        qa_task._ask = MagicMock(return_value={"response": "test answer"})
-        result = qa_task.ask("test question")
-        qa_task._ask.assert_called_once()
-        self.assertEqual(result.get("response"), "test answer")
-        self.assertIsNotNone(result.get("session_id"))
-
-    @patch.object(QaTask, "_build_chain")
-    def test_ask_without_session(self, mock_build_chain):
-        mock_build_chain.return_value = MagicMock()
-        qa_task = QaTask()
-        qa_task._ask = MagicMock(return_value={"answer": "test answer"})
-        result = qa_task.ask("test question", session=False)
-        qa_task._ask.assert_called_once_with("test question")
-        self.assertEqual(result, {"answer": "test answer"})
-
-    @patch.object(QaTask, "_build_chain")
-    def test__get_session_mem(self, mock_build_chain):
-        mock_build_chain.return_value = MagicMock()
-        memory = ConversationBufferMemory()
-        qa_task = QaTask(session_store={1: memory})
-        result = qa_task._get_session_mem(1)
-        self.assertEqual(result, memory)
-
-    @patch.object(QaTask, "_build_chain")
-    def test__update_session_mem(self, mock_build_chain):
-        mock_build_chain.return_value = MagicMock()
-        memory = ConversationBufferMemory()
-        qa_task = QaTask()
-        qa_task._update_session_mem(1, memory)
-        self.assertEqual(qa_task._session_store[1], memory)
-
-
-if __name__ == "__main__":
-    unittest.main()