Spaces:
Sleeping
Sleeping
| """Evaluator Agent -> Idea-Observer. | |
| The model acts as an idea-observer. It evaluates the student's ideas against the | |
| source material based on their questions, quiz answers, and Feynman explanations. | |
| It produces a grounded prose observation rather than numeric scores. | |
| """ | |
| from typing import List, Optional | |
| from pydantic import BaseModel, Field | |
| from app.agents.cerebras_client import CerebrasClient | |
| from app.services.journal_service import JournalService | |
| class NodeAssessment(BaseModel): | |
| node_id: str | |
| observation: str = Field( | |
| description="A short cited prose observation about the student's idea or understanding of this concept, and how it held up against the source material." | |
| ) | |
| evidence: List[str] = Field(default_factory=list, description="0-3 specific signals/quotes from the journal to ground the observation") | |
| source_evidence_ids: List[str] = Field( | |
| default_factory=list, | |
| description="0-4 exact ev_ IDs from SOURCE EVIDENCE that support the observation", | |
| ) | |
| class _EvaluatorOutput(BaseModel): | |
| assessments: List[NodeAssessment] | |
| session_summary: str = Field(description="One-paragraph plain-text summary of the session") | |
| class EvaluatorAgent: | |
| def __init__( | |
| self, | |
| journal_service: Optional[JournalService] = None, | |
| client: Optional[CerebrasClient] = None, | |
| ) -> None: | |
| self._client = client or CerebrasClient() | |
| self._journal = journal_service or JournalService() | |
| def evaluate_session( | |
| self, | |
| project_id: str, | |
| prior_context: str = "", | |
| source_context: str = "", | |
| ) -> tuple[List[NodeAssessment], str]: | |
| """Returns (assessments, summary_text). | |
| prior_context is this student's cross-session Cognee memory (from | |
| query_prior_knowledge). The idea-observer reads it so each observation is | |
| judged against the TRAJECTORY -> "continues to conflate X as in an earlier | |
| session" / "has moved from what-is to why questions here" -> not just the | |
| current session's journal in isolation. This is what makes the Evaluator a | |
| memory READER, one of Tier 1A's load-bearing tenets, not a writer alone. | |
| """ | |
| journal = self._journal.get_session(project_id) | |
| if not journal: | |
| return [], "No activity recorded in this session." | |
| journal_text = "\n".join( | |
| f"[{e.event_type}] node={e.node_id} data={e.data}" for e in journal | |
| ) | |
| trajectory_note = "" | |
| if prior_context: | |
| trajectory_note = ( | |
| "\n\nPRIOR-SESSION MEMORY OF THIS STUDENT (judge against this trajectory, don't " | |
| "restate it): note where an idea has PERSISTED, DEEPENED, or been CORRECTED since " | |
| "earlier sessions, and call out recurring gaps explicitly.\n" | |
| f"{prior_context}" | |
| ) | |
| messages = [ | |
| { | |
| "role": "system", | |
| "content": ( | |
| "You are an idea-observer. For each concept node in the journal, " | |
| "evaluate the student's ideas and understanding against the source material based " | |
| "on their questions, quiz answers, and Feynman explanations. " | |
| "Write a short cited prose observation about their idea and how it held up. " | |
| "Do NOT grade the student or output numeric scores. " | |
| "Focus on the ideas themselves (e.g., 'Proposed X analogy; holds up well but breaks on Y'). " | |
| "For every paper-grounded judgment, include only exact ev_ identifiers from the " | |
| "provided SOURCE EVIDENCE in source_evidence_ids; never invent an ID. Journal evidence " | |
| "and source evidence are separate. Also write a one-paragraph session_summary capturing " | |
| "their overall learning trajectory." | |
| ), | |
| }, | |
| { | |
| "role": "user", | |
| "content": ( | |
| f"Session journal:\n{journal_text}{trajectory_note}\n\n" | |
| f"SOURCE EVIDENCE FROM UPLOADED PAPERS:\n{source_context or 'No source evidence available.'}\n\n" | |
| "Observe each node's ideas." | |
| ), | |
| }, | |
| ] | |
| output = self._client.structured_complete(messages, _EvaluatorOutput) | |
| return output.assessments, output.session_summary | |