Syntagma
+ +Sign in to continue
+diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..e7acb09b33996bce5f1579f406f81dbc0ffebed3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libgdk-pixbuf-2.0-0 \ + libffi-dev \ + poppler-utils \ + shared-mime-info \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY backend/ . +COPY frontend/ frontend/ + +EXPOSE 7860 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"] diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..71cf14c84788c1bd304a9f86d53110282e6e7043 --- /dev/null +++ b/README.md @@ -0,0 +1,205 @@ +--- +title: Curriculum Backend +colorFrom: blue +colorTo: gray +sdk: docker +app_port: 7860 +pinned: false +--- + +# Syntagma + +
| {_esc(str(cell) if cell is not None else '')} | ") + parts.append("
|---|
| {_esc(str(cell) if cell is not None else '')} | ") + parts.append("
\1", html)
+ # Code blocks
+ html = re.sub(
+ r"```(\w+)?\n(.+?)```",
+ r'\2',
+ html,
+ flags=re.DOTALL,
+ )
+ # Links
+ html = re.sub(r"\[(.+?)\]\((.+?)\)", r'\1', html)
+ # Unordered lists
+ html = re.sub(r"^\- (.+)$", r"") + in_p = True + result.append(line) + else: + if in_p: + result.append("
") + in_p = False + result.append(line) + if in_p: + result.append("") + return "\n".join(result) + + +ToolHandler = Callable[[dict], dict] + + +@dataclass(frozen=True) +class AgentTool: + name: str + description: str + parameters: dict + handler: ToolHandler + + def schema(self) -> dict: + return { + "type": "function", + "function": { + "name": self.name, + "description": self.description, + "parameters": self.parameters, + }, + } + + +def list_tool_schemas() -> list[dict]: + return [tool.schema() for tool in TOOLS.values()] + + +def call_tool(name: str, arguments: dict | None = None) -> dict: + tool = TOOLS.get(name) + if not tool: + raise LookupError("Agent tool not found") + return tool.handler(arguments or {}) + + +def _require_int(arguments: dict, key: str) -> int: + value = arguments.get(key) + if value is None: + raise ValueError(f"{key} is required") + return int(value) + + +def _require_dict(arguments: dict, key: str) -> dict: + value = arguments.get(key) + if not isinstance(value, dict): + raise ValueError(f"{key} must be an object") + return value + + +def _get_current_course(arguments: dict) -> dict: + return {"course": refined_course(_require_int(arguments, "refined_id"))} + + +def _get_course_fields(arguments: dict) -> dict: + """Get specific field groups from a course. More efficient than full course JSON.""" + refined_id = _require_int(arguments, "refined_id") + fields = arguments.get("fields") + if not isinstance(fields, list) or not fields: + raise ValueError("fields must be a non-empty array of field names") + + course = refined_course(refined_id) + result = {"refined_id": refined_id} + for field in fields: + if field in course: + result[field] = course[field] + else: + result[field] = None + return result + + +def _get_course_codes(arguments: dict) -> dict: + """Get lightweight course identifiers: refined_id, course_code, course_title, semester.""" + refined_id = _require_int(arguments, "refined_id") + course = refined_course(refined_id) + return { + "refined_id": refined_id, + "course_code": course.get("course_code"), + "course_title": course.get("course_title"), + "semester": course.get("semester"), + "program": course.get("program"), + } + + +def _get_course_syllabus(arguments: dict) -> dict: + """Get syllabus content: units, objectives, course_outcomes.""" + refined_id = _require_int(arguments, "refined_id") + course = refined_course(refined_id) + return { + "refined_id": refined_id, + "units": course.get("units"), + "objectives": course.get("objectives"), + "course_outcomes": course.get("course_outcomes"), + } + + +def _get_course_textbooks(arguments: dict) -> dict: + """Get textbook fields: text_books, reference_books.""" + refined_id = _require_int(arguments, "refined_id") + course = refined_course(refined_id) + return { + "refined_id": refined_id, + "text_books": course.get("text_books"), + "reference_books": course.get("reference_books"), + } + + +def _get_course_deterministic(arguments: dict) -> dict: + """Get deterministic fields (program, hours, credits, course_type) — these are agent-protected.""" + refined_id = _require_int(arguments, "refined_id") + course = refined_course(refined_id) + return { + "refined_id": refined_id, + "program": course.get("program"), + "lecture_hours": course.get("lecture_hours"), + "tutorial_hours": course.get("tutorial_hours"), + "practical_hours": course.get("practical_hours"), + "self_study": course.get("self_study"), + "credits": course.get("credits"), + "course_type": course.get("course_type"), + } + + +def _get_course_lab(arguments: dict) -> dict: + """Get lab experiments and tools/languages.""" + refined_id = _require_int(arguments, "refined_id") + course = refined_course(refined_id) + return { + "refined_id": refined_id, + "lab_experiments": course.get("lab_experiments"), + "tools_languages": course.get("tools_languages"), + } + + +def _diff_course_json(arguments: dict) -> dict: + return diff_course( + _require_dict(arguments, "current"), _require_dict(arguments, "proposed") + ) + + +def _create_course_draft(arguments: dict) -> dict: + refined_id = _require_int(arguments, "refined_id") + if refined_id <= 0: + raise ValueError( + "refined_id must be a valid existing course ID. To create a brand-new course, use create_refined_course instead." + ) + fields = arguments.get("fields") + if not isinstance(fields, dict) or not fields: + raise ValueError( + 'fields must be a non-empty object containing only the fields to change, e.g. {"text_books": "new value"}. Do not pass all course data; only pass what should change.' + ) + for key in _ARRAY_FIELDS: + if key in fields: + fields[key] = _coerce_array(fields[key]) + row = first_row(supabase.table("refined_submissions").select("status").eq("id", refined_id)) + if not row: + raise ValueError(f"Course with refined_id {refined_id} not found.") + if row.get("status") == "draft": + raise ValueError(f"Course {refined_id} has status 'draft'. Use create_refined_course with the refined_id to update draft-status courses directly.") + record = draft_record(refined_id, fields, str(arguments.get("reason") or "")) + draft = supabase.table("agent_drafts").insert(record).execute().data[0] + return {"draft": draft} + + +def _update_agent_draft(arguments: dict) -> dict: + draft_id = _require_int(arguments, "draft_id") + fields = arguments.get("fields") + if not isinstance(fields, dict) or not fields: + raise ValueError( + 'fields must be a non-empty object containing only the fields to change, e.g. {"text_books": "new value"}.' + ) + for key in _ARRAY_FIELDS: + if key in fields: + fields[key] = _coerce_array(fields[key]) + row = first_row(supabase.table("agent_drafts").select("refined_id,proposed_json,status").eq("id", draft_id)) + if not row: + raise ValueError(f"Draft with id {draft_id} not found.") + if row.get("status") not in ("proposed", "blocked"): + raise ValueError(f"Draft {draft_id} has status '{row.get('status')}' and cannot be edited. Only proposed or blocked drafts can be updated.") + from app.services.curriculum import merge_fields, diff_course, validate_draft + proposed = merge_fields(row["proposed_json"], fields) + base = first_row(supabase.table("refined_submissions").select("*").eq("id", int(row["refined_id"]))) + base = base or {} + summary = diff_course(base, proposed) + issues = validate_draft(base, proposed) + summary["validation_issues"] = issues + update = { + "proposed_json": proposed, + "json_patch": summary.pop("json_patch"), + "diff_summary": summary, + "status": "blocked" if issues else "proposed", + } + result = supabase.table("agent_drafts").update(update).eq("id", draft_id).execute() + return {"draft": result.data[0] if result.data else row} + + +_ARRAY_FIELDS = { + "course_outcomes", + "lab_experiments", + "objectives", + "text_books", + "reference_books", + "units", +} + + +def _coerce_array(value): + if isinstance(value, list): + return value + if isinstance(value, str): + stripped = value.strip() + if stripped.startswith("["): + try: + parsed = json.loads(stripped) + if isinstance(parsed, list): + return parsed + except json.JSONDecodeError: + pass + lines = [line.strip() for line in stripped.splitlines()] + cleaned = [] + for line in lines: + for prefix in ("- ", "• ", "* "): + if line.startswith(prefix): + line = line[len(prefix):] + break + line = line.strip() + if line: + cleaned.append(line) + if len(cleaned) > 1: + return cleaned + return [stripped] if stripped else [] + return [value] if value is not None else [] + + +def _create_refined_course(arguments: dict) -> dict: + from app.services.deterministic import ( + compute_course_type, + compute_hours, + compute_program, + ) + + credit_category = str(arguments.get("credit_category") or "4") + target_dept = str(arguments.get("target_department") or "CSE") + det = compute_hours(credit_category) + + is_elective = bool(arguments.get("is_elective")) + computed = { + "program": compute_program(target_dept), + "course_type": "Elective" if is_elective else compute_course_type(credit_category), + "status": "draft", + } + for key in ( + "lecture_hours", + "tutorial_hours", + "practical_hours", + "self_study", + "credits", + ): + if key in arguments: + computed[key] = int(arguments[key] or 0) + else: + computed[key] = det[key] + if "semester" in arguments: + computed["semester"] = int(arguments["semester"] or 0) + + fields = { + k: v for k, v in arguments.items() if v is not None and k in REFINED_FIELDS + } + for key in _ARRAY_FIELDS: + if key in fields: + fields[key] = _coerce_array(fields[key]) + dk = str(fields.get("desirable_knowledge") or "").strip() + if dk and dk.lower() in ("none", "n/a", "na", "-"): + fields["desirable_knowledge"] = "" + if "is_elective" in arguments: + fields["is_elective"] = bool(arguments["is_elective"]) + fields.update(computed) + + refined_id = arguments.get("refined_id") + if not refined_id and "course_code" in fields: + dup = ( + supabase.table("refined_submissions") + .select("id") + .eq("course_code", fields["course_code"]) + .limit(1) + .execute() + .data + ) + if dup: + return {"error": f"Course code {fields['course_code']} already exists (refined_id: {dup[0]['id']}). Pick a unique code."} + + if refined_id: + result = ( + supabase.table("refined_submissions") + .update(fields) + .eq("id", int(refined_id)) + .execute() + ) + row = result.data[0] if result.data else None + invalidate_curriculum_cache() + return {"refined_id": int(refined_id), "updated": True, "course": row} + + if "submission_id" not in fields or not fields.get("submission_id"): + placeholder = ( + supabase.table("submissions") + .insert( + { + "faculty_email": "ai-generated@pes.edu", + "course_title": fields.get("course_title") or "", + "offering_department": "CS", + "target_department": arguments.get("target_department") or "CSE", + "semester": int(arguments.get("semester") or 1), + "credit_category": arguments.get("credit_category") or "4", + "raw_course_content": fields.get("prelude") or "AI-created course", + "text_books": "", + "reference_books": "", + "preferred_tools": "", + "status": "refined", + } + ) + .execute() + .data[0] + ) + fields["submission_id"] = placeholder["id"] + + result = supabase.table("refined_submissions").insert(fields).execute() + row = result.data[0] + invalidate_curriculum_cache() + return {"refined_id": row["id"], "updated": False, "course": row} + + +def _get_curriculum_json(arguments: dict) -> dict: + query = supabase.table("refined_submissions").select("*").in_("status", ["refined"]) + if arguments.get("semester") is not None: + query = query.eq("semester", int(arguments["semester"])) + return {"courses": ordered_courses(query.execute().data)} + + +def _create_document_draft(arguments: dict) -> dict: + courses = arguments.get("courses") + if not isinstance(courses, list) or not courses: + raise ValueError("courses must be a non-empty array") + + records = [] + for course in courses: + if not isinstance(course, dict): + raise ValueError("each course must be an object") + fields = _require_dict(course, "fields") + for key in _ARRAY_FIELDS: + if key in fields: + fields[key] = _coerce_array(fields[key]) + records.append( + draft_record( + int(course.get("refined_id")), + fields, + str(arguments.get("reason") or ""), + ) + ) + + summaries = [record["diff_summary"] for record in records] + document_summary = { + "courses_changed": len(records), + "courses_with_removed_topics": sum( + 1 for summary in summaries if summary.get("topics_removed") + ), + "courses_with_protected_changes": sum( + 1 for summary in summaries if summary.get("protected_changes") + ), + "max_syllabus_change_percent": max( + (summary.get("syllabus_change_percent") or 0 for summary in summaries), + default=0, + ), + } + document = ( + supabase.table("agent_document_drafts") + .insert( + { + "curriculum_version_id": arguments.get("curriculum_version_id"), + "uploaded_document_id": str( + arguments.get("uploaded_document_id") or "" + ).strip(), + "diff_summary": document_summary, + "change_reason": str(arguments.get("reason") or "").strip(), + "status": "blocked" + if document_summary["courses_with_protected_changes"] + else "proposed", + } + ) + .execute() + .data[0] + ) + for record in records: + record["document_draft_id"] = document["id"] + drafts = supabase.table("agent_drafts").insert(records).execute().data + return {"document_draft": document, "drafts": drafts} + + +def _get_course_draft(arguments: dict) -> dict: + return {"draft": load_agent_draft(_require_int(arguments, "draft_id"))} + + +def _get_document_draft(arguments: dict) -> dict: + return load_document_draft(_require_int(arguments, "document_draft_id")) + + +def _get_preview_url(arguments: dict) -> dict: + kind = str(arguments.get("kind") or "") + item_id = _require_int(arguments, "id") + paths = { + "course": f"/api/preview/course/{item_id}", + "draft": f"/api/agent/drafts/{item_id}/preview", + "document_draft": f"/api/agent/document-drafts/{item_id}/preview", + } + if kind not in paths: + raise ValueError("kind must be course, draft, or document_draft") + return {"url": paths[kind]} + + +def _list_courses(arguments: dict) -> dict: + query = ( + supabase.table("refined_submissions") + .select("id,semester,course_code,course_title") + .neq("status", "archived") + ) + if arguments.get("semester") is not None: + query = query.eq("semester", int(arguments["semester"])) + rows = query.execute().data + rows.sort( + key=lambda row: ( + int(row.get("semester") or 0), + str(row.get("course_code") or ""), + int(row.get("id") or 0), + ) + ) + return {"courses": rows} + + +def _fetch_url(arguments: dict) -> dict: + url = str(arguments.get("url") or "").strip() + if not url: + raise ValueError("url is required") + resp = httpx.get(url, timeout=30, follow_redirects=True) + resp.raise_for_status() + text = resp.text[:15000] + return {"url": url, "text": text, "chars": len(text)} + + +def _web_search(arguments: dict) -> dict: + """Search the web using DuckDuckGo's HTML endpoint and return top results.""" + query = str(arguments.get("query") or "").strip() + if not query: + raise ValueError("query is required") + num_results = int(arguments.get("num_results") or 5) + num_results = min(max(num_results, 1), 10) + + url = f"https://html.duckduckgo.com/html/?q={quote_plus(query)}" + resp = httpx.get( + url, timeout=15, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"} + ) + resp.raise_for_status() + + # Parse results from DuckDuckGo HTML + html = resp.text + results = [] + # Pattern for result snippets + result_pattern = re.compile(r'class="result__snippet">(.*?)', re.DOTALL) + title_pattern = re.compile(r'class="result__title">.*?>(.*?)', re.DOTALL) + url_pattern = re.compile(r'class="result__url">.*?>(.*?)', re.DOTALL) + + snippets = result_pattern.findall(html) + titles = title_pattern.findall(html) + urls = url_pattern.findall(html) + + for i in range(min(num_results, len(snippets))): + title = re.sub(r"<[^>]+>", "", titles[i] if i < len(titles) else "").strip() + snippet = re.sub(r"<[^>]+>", "", snippets[i]).strip() + link = re.sub(r"<[^>]+>", "", urls[i] if i < len(urls) else "").strip() + if title or snippet: + results.append({"title": title, "snippet": snippet[:300], "url": link}) + + return {"query": query, "results": results} + + +def _create_report(arguments: dict) -> dict: + session_id = _require_int(arguments, "session_id") + content = str(arguments.get("content") or "").strip() + if not content: + raise ValueError("content is required") + filename = str(arguments.get("filename") or "report.md").strip() + fmt = str(arguments.get("format") or "markdown").strip().lower() + if fmt not in ("markdown", "pdf"): + raise ValueError("format must be 'markdown' or 'pdf'") + + if fmt == "pdf": + # Convert markdown to HTML then to PDF + # Simple markdown to HTML conversion + html_content = _markdown_to_html(content) + html_full = f""" + + + + + + +{html_content} + +""" + + # Use weasyprint to generate PDF + from weasyprint import HTML + + pdf_bytes = HTML(string=html_full, base_url=".").write_pdf() + + if filename.endswith(".md"): + filename = filename[:-3] + ".pdf" + elif not filename.endswith(".pdf"): + filename = filename + ".pdf" + + content_type = "application/pdf" + size_bytes = len(pdf_bytes) + extracted_text = f"[PDF file - {size_bytes} bytes]" + content_base64 = base64.b64encode(pdf_bytes).decode() + else: + pdf_bytes = None + content_type = "text/markdown" + size_bytes = len(content.encode()) + extracted_text = content + content_base64 = "" + + row = ( + supabase.table("chat_attachments") + .insert( + { + "session_id": session_id, + "filename": filename, + "content_type": content_type, + "size_bytes": size_bytes, + "extracted_text": extracted_text, + "content_base64": content_base64, + "status": "ready", + } + ) + .execute() + .data[0] + ) + + return { + "attachment": { + "id": row["id"], + "filename": row["filename"], + "chars": size_bytes, + "format": fmt, + } + } + + +def _attachment_text(arguments: dict) -> dict: + session_id = _require_int(arguments, "session_id") + ids = [int(value) for value in arguments.get("attachment_ids") or []] + if not ids: + raise ValueError("attachment_ids is required") + rows = ( + supabase.table("chat_attachments") + .select("id,filename,status,error,extracted_text") + .eq("session_id", session_id) + .in_("id", ids) + .execute() + .data + ) + return {"attachments": rows} + + +def _create_curriculum_version(arguments: dict) -> dict: + name = str(arguments.get("name") or "").strip() + if not name: + raise ValueError("name is required") + version = create_version_snapshot(name) + return {"version": version} + + +def _define_specialization(arguments: dict) -> dict: + semester = _require_int(arguments, "semester") + name = str(arguments.get("name") or "").strip() + if not name: + raise ValueError("name is required") + letter = str(arguments.get("letter") or "").strip().upper() + if not letter: + existing = ( + supabase.table("specialization_definitions") + .select("letter") + .eq("semester", semester) + .execute() + .data + ) + used = {row["letter"] for row in existing} + for candidate in "ABCDEFGHIJKLMNOPQRSTUVWXYZ": + if candidate not in used: + letter = candidate + break + if not letter: + raise ValueError( + "No free specialization letter available for this semester" + ) + academic_year = str(arguments.get("academic_year") or "").strip() + key = str(arguments.get("key") or "").strip().upper() + if not key: + key = name.split("(")[-1].rstrip(")") if "(" in name else name[:3].upper() + row = ( + supabase.table("specialization_definitions") + .insert( + { + "semester": semester, + "letter": letter, + "name": name, + "key": key, + "academic_year": academic_year, + } + ) + .execute() + .data[0] + ) + return {"specialization": row} + + +def _list_specializations(arguments: dict) -> dict: + query = supabase.table("specialization_definitions").select("*") + if arguments.get("semester") is not None: + query = query.eq("semester", int(arguments["semester"])) + return {"specializations": query.order("semester").order("letter").execute().data} + + +def _assign_elective_to_tracks(arguments: dict) -> dict: + refined_id = _require_int(arguments, "refined_id") + spec_ids = arguments.get("specialization_ids") + if not isinstance(spec_ids, list) or not spec_ids: + raise ValueError("specialization_ids must be a non-empty array") + created = 0 + for spec_id in spec_ids: + existing = ( + supabase.table("course_specialization_assignments") + .select("id") + .eq("refined_id", refined_id) + .eq("specialization_id", int(spec_id)) + .execute() + .data + ) + if not existing: + supabase.table("course_specialization_assignments").insert( + {"refined_id": refined_id, "specialization_id": int(spec_id)} + ).execute() + created += 1 + supabase.table("refined_submissions").update({"is_elective": True}).eq( + "id", refined_id + ).execute() + return {"assignments_created": created} + + +def _remove_elective_from_tracks(arguments: dict) -> dict: + refined_id = _require_int(arguments, "refined_id") + spec_ids = arguments.get("specialization_ids") + if not isinstance(spec_ids, list) or not spec_ids: + raise ValueError("specialization_ids must be a non-empty array") + removed = 0 + for spec_id in spec_ids: + result = ( + supabase.table("course_specialization_assignments") + .delete() + .eq("refined_id", refined_id) + .eq("specialization_id", int(spec_id)) + .execute() + ) + removed += len(result.data or []) + return {"assignments_removed": removed} + + +def _get_course_assignments(arguments: dict) -> dict: + refined_id = _require_int(arguments, "refined_id") + assignments = ( + supabase.table("course_specialization_assignments") + .select("*, specialization_definitions(*)") + .eq("refined_id", refined_id) + .execute() + .data + ) + return {"refined_id": refined_id, "assignments": assignments} + + +def _categorize_elective(arguments: dict) -> dict: + """Re-run guarded AI categorization for an elective needing human review.""" + from app.services.elective_categorization import categorize_refined_elective + + return categorize_refined_elective(_require_int(arguments, "refined_id")) + + +def _update_deterministic_fields(arguments: dict) -> dict: + from app.services.diffing import PROTECTED_FIELDS + + refined_id = _require_int(arguments, "refined_id") + fields = _require_dict(arguments, "fields") + protected = {key: value for key, value in fields.items() if key in PROTECTED_FIELDS} + if not protected: + raise ValueError( + "No deterministic fields provided. Use create_course_draft for other fields." + ) + record = draft_record( + refined_id, + protected, + str( + arguments.get("reason") + or "Explicit user request to change deterministic fields" + ), + ) + record["status"] = "blocked" + draft = supabase.table("agent_drafts").insert(record).execute().data[0] + return { + "draft": draft, + "warning": "This draft modifies deterministic fields (program, hours, credits, course_type). The user must explicitly approve it in the Review panel before it is applied.", + } + + +def _signal_done(arguments: dict) -> dict: + summary = str(arguments.get("summary") or "").strip() + if not summary: + raise ValueError("summary is required") + return {"done": True, "summary": summary} + + +def _ask_user(arguments: dict) -> dict: + question = str(arguments.get("question") or "").strip() + if not question: + raise ValueError("question is required") + return {"asked": True, "question": question, "done": True, "summary": question} + + +def _create_spreadsheet(arguments: dict) -> dict: + """Generate CSV or Excel file from rows data and save as chat attachment.""" + session_id = _require_int(arguments, "session_id") + rows = arguments.get("rows") + if not isinstance(rows, list) or not rows: + raise ValueError("rows must be a non-empty array of objects") + columns = arguments.get("columns") + if not isinstance(columns, list) or not columns: + raise ValueError("columns must be a non-empty array of column names") + filename = str(arguments.get("filename") or "spreadsheet.csv").strip() + fmt = str(arguments.get("format") or "csv").strip().lower() + if fmt not in ("csv", "xlsx"): + raise ValueError("format must be 'csv' or 'xlsx'") + + if fmt == "xlsx": + import io + from openpyxl import Workbook + from openpyxl.styles import Font, PatternFill, Alignment, Border, Side + + wb = Workbook() + ws = wb.active + ws.title = "Data" + + header_font = Font(bold=True, color="FFFFFF", size=11) + header_fill = PatternFill( + start_color="00377B", end_color="00377B", fill_type="solid" + ) + thin_border = Border( + left=Side(style="thin"), + right=Side(style="thin"), + top=Side(style="thin"), + bottom=Side(style="thin"), + ) + + for col_idx, col_name in enumerate(columns, 1): + cell = ws.cell(row=1, column=col_idx, value=col_name) + cell.font = header_font + cell.fill = header_fill + cell.alignment = Alignment(horizontal="center") + cell.border = thin_border + + for row_idx, row_data in enumerate(rows, 2): + for col_idx, col_name in enumerate(columns, 1): + value = row_data.get(col_name, "") + cell = ws.cell(row=row_idx, column=col_idx, value=value) + cell.border = thin_border + cell.alignment = Alignment(wrap_text=True) + + for col_idx, col_name in enumerate(columns, 1): + max_len = len(str(col_name)) + for row_idx in range(2, len(rows) + 2): + val = str(ws.cell(row=row_idx, column=col_idx).value or "") + max_len = max(max_len, min(len(val), 60)) + ws.column_dimensions[ws.cell(row=1, column=col_idx).column_letter].width = ( + max_len + 2 + ) + + buf = io.BytesIO() + wb.save(buf) + buf.seek(0) + file_bytes = buf.read() + + if not filename.endswith(".xlsx"): + filename = filename.rsplit(".", 1)[0] + ".xlsx" + content_type = ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + extracted_text = f"[Excel file - {len(file_bytes)} bytes, {len(rows)} rows]" + content_base64 = base64.b64encode(file_bytes).decode() + size_bytes = len(file_bytes) + else: + import io + import csv + + buf = io.StringIO() + writer = csv.DictWriter(buf, fieldnames=columns, extrasaction="ignore") + writer.writeheader() + for row_data in rows: + writer.writerow({k: v for k, v in row_data.items() if k in columns}) + csv_text = buf.getvalue() + + if not filename.endswith(".csv"): + filename = filename.rsplit(".", 1)[0] + ".csv" + content_type = "text/csv" + extracted_text = csv_text + content_base64 = "" + size_bytes = len(csv_text.encode()) + + row = ( + supabase.table("chat_attachments") + .insert( + { + "session_id": session_id, + "filename": filename, + "content_type": content_type, + "size_bytes": size_bytes, + "extracted_text": extracted_text, + "content_base64": content_base64, + "status": "ready", + } + ) + .execute() + .data[0] + ) + return { + "attachment": { + "id": row["id"], + "filename": row["filename"], + "rows": len(rows), + "columns": columns, + "format": fmt, + } + } + + +def _get_version(arguments: dict) -> dict: + """Load a curriculum version snapshot with its course list.""" + version_id = _require_int(arguments, "version_id") + version_row = first_row( + supabase.table("curriculum_versions").select("*").eq("id", version_id) + ) + if not version_row: + raise ValueError(f"Version {version_id} not found") + snapshot_rows = ( + supabase.table("finalized_submissions") + .select("refined_id,course_json") + .eq("curriculum_version_id", version_id) + .order("refined_id") + .execute() + .data + ) + courses = [] + for snap in snapshot_rows: + cj = snap.get("course_json") or {} + courses.append( + { + "refined_id": snap.get("refined_id"), + "course_code": cj.get("course_code", ""), + "course_title": cj.get("course_title", ""), + "semester": cj.get("semester", ""), + "credits": cj.get("credits", ""), + } + ) + return {"version": version_row, "courses": courses, "course_count": len(courses)} + + +def _diff_versions(arguments: dict) -> dict: + """Compare two curriculum version snapshots. Shows added, removed, and changed courses.""" + version_a_id = _require_int(arguments, "version_a") + version_b_id = _require_int(arguments, "version_b") + + def _load_snapshot(vid: int) -> dict: + rows = ( + supabase.table("finalized_submissions") + .select("refined_id,course_json") + .eq("curriculum_version_id", vid) + .execute() + .data + ) + return {row["refined_id"]: row.get("course_json") or {} for row in rows} + + snap_a = _load_snapshot(version_a_id) + snap_b = _load_snapshot(version_b_id) + + all_ids = set(snap_a.keys()) | set(snap_b.keys()) + added = [] + removed = [] + changed = [] + unchanged = [] + + for rid in sorted(all_ids): + a = snap_a.get(rid) + b = snap_b.get(rid) + if a and not b: + removed.append( + { + "refined_id": rid, + "course_code": a.get("course_code", ""), + "course_title": a.get("course_title", ""), + } + ) + elif b and not a: + added.append( + { + "refined_id": rid, + "course_code": b.get("course_code", ""), + "course_title": b.get("course_title", ""), + } + ) + elif a != b: + d = diff_course(a, b) + changed.append( + { + "refined_id": rid, + "course_code": b.get("course_code", ""), + "course_title": b.get("course_title", ""), + "change_percent": d.get("change_percent", 0), + "syllabus_change_percent": d.get("syllabus_change_percent", 0), + "topics_added": d.get("topics_added", []), + "topics_removed": d.get("topics_removed", []), + } + ) + else: + unchanged.append(rid) + + return { + "version_a": version_a_id, + "version_b": version_b_id, + "added": added, + "removed": removed, + "changed": changed, + "unchanged_count": len(unchanged), + "summary": { + "added": len(added), + "removed": len(removed), + "changed": len(changed), + "unchanged": len(unchanged), + }, + } + + +def _get_curriculum_stats(arguments: dict) -> dict: + """Compute aggregate statistics for the curriculum or a specific semester.""" + query = ( + supabase.table("refined_submissions") + .select( + "semester,credits,course_type,credit_category,lecture_hours,practical_hours,visible,is_elective" + ) + .in_("status", ["refined"]) + ) + if arguments.get("semester") is not None: + query = query.eq("semester", int(arguments["semester"])) + rows = query.execute().data + + if not rows: + return {"stats": {}, "message": "No courses found"} + + by_semester: dict[int, dict] = {} + for row in rows: + sem = int(row.get("semester") or 0) + if sem not in by_semester: + by_semester[sem] = { + "total": 0, + "visible": 0, + "total_credits": 0, + "electives": 0, + "course_types": {}, + "credit_categories": {}, + } + s = by_semester[sem] + s["total"] += 1 + if row.get("visible", True): + s["visible"] += 1 + s["total_credits"] += int(row.get("credits") or 0) + if row.get("is_elective"): + s["electives"] += 1 + ct = str(row.get("course_type") or "Unknown") + s["course_types"][ct] = s["course_types"].get(ct, 0) + 1 + cc = str(row.get("credit_category") or "?") + s["credit_categories"][cc] = s["credit_categories"].get(cc, 0) + 1 + + total_credits = sum(s["total_credits"] for s in by_semester.values()) + total_courses = sum(s["total"] for s in by_semester.values()) + + return { + "total_courses": total_courses, + "total_credits": total_credits, + "by_semester": {str(k): v for k, v in sorted(by_semester.items())}, + } + + +def _batch_read_courses(arguments: dict) -> dict: + """Read specific fields from multiple courses in one call. More efficient than multiple get_course_fields calls.""" + refined_ids = arguments.get("refined_ids") + if not isinstance(refined_ids, list) or not refined_ids: + raise ValueError("refined_ids must be a non-empty array of integers") + fields = arguments.get("fields") + if not isinstance(fields, list) or not fields: + raise ValueError("fields must be a non-empty array of field names") + + results = [] + for rid in refined_ids: + rid = int(rid) + course = refined_course(rid) + if course is None: + results.append({"refined_id": rid, "error": "Course not found"}) + continue + entry = {"refined_id": rid} + for field in fields: + entry[field] = course.get(field) + results.append(entry) + + return {"courses": results, "count": len(results)} + + +OBJECT = {"type": "object", "additionalProperties": False} + +TOOLS: dict[str, AgentTool] = { + "get_current_course_json": AgentTool( + "get_current_course_json", + "Read the current template-ready JSON for one refined course.", + { + **OBJECT, + "properties": {"refined_id": {"type": "integer"}}, + "required": ["refined_id"], + }, + _get_current_course, + ), + "get_course_codes": AgentTool( + "get_course_codes", + "Read lightweight course identifiers (refined_id, course_code, course_title, semester, program). Use for listing or quick lookups.", + { + **OBJECT, + "properties": {"refined_id": {"type": "integer"}}, + "required": ["refined_id"], + }, + _get_course_codes, + ), + "get_course_syllabus": AgentTool( + "get_course_syllabus", + "Read syllabus content: units, objectives, course_outcomes.", + { + **OBJECT, + "properties": {"refined_id": {"type": "integer"}}, + "required": ["refined_id"], + }, + _get_course_syllabus, + ), + "get_course_textbooks": AgentTool( + "get_course_textbooks", + "Read textbook fields: text_books, reference_books.", + { + **OBJECT, + "properties": {"refined_id": {"type": "integer"}}, + "required": ["refined_id"], + }, + _get_course_textbooks, + ), + "get_course_deterministic": AgentTool( + "get_course_deterministic", + "Read deterministic/protected fields: program, lecture_hours, tutorial_hours, practical_hours, self_study, credits, course_type. These cannot be changed by the agent.", + { + **OBJECT, + "properties": {"refined_id": {"type": "integer"}}, + "required": ["refined_id"], + }, + _get_course_deterministic, + ), + "get_course_lab": AgentTool( + "get_course_lab", + "Read lab experiments and tools/languages.", + { + **OBJECT, + "properties": {"refined_id": {"type": "integer"}}, + "required": ["refined_id"], + }, + _get_course_lab, + ), + "get_course_fields": AgentTool( + "get_course_fields", + "Read arbitrary specific fields from a course. Provide a list of field names. More efficient than fetching full JSON when you only need a subset.", + { + **OBJECT, + "properties": { + "refined_id": {"type": "integer"}, + "fields": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + }, + "required": ["refined_id", "fields"], + }, + _get_course_fields, + ), + "diff_course_json": AgentTool( + "diff_course_json", + "Compare two course JSON objects and return patch operations, changed percent, and syllabus topic changes.", + { + **OBJECT, + "properties": { + "current": {"type": "object"}, + "proposed": {"type": "object"}, + }, + "required": ["current", "proposed"], + }, + _diff_course_json, + ), + "create_course_draft": AgentTool( + "create_course_draft", + "Create a human-reviewable draft for one course. This never applies changes to refined_submissions.", + { + **OBJECT, + "properties": { + "refined_id": {"type": "integer"}, + "fields": {"type": "object"}, + "reason": {"type": "string"}, + }, + "required": ["refined_id", "fields"], + }, + _create_course_draft, + ), + "update_agent_draft": AgentTool( + "update_agent_draft", + "Update an existing proposed or blocked draft with new field values. Use this to modify a draft you already created instead of creating a duplicate. Merges the new fields into the existing proposed_json.", + { + **OBJECT, + "properties": { + "draft_id": {"type": "integer", "description": "The ID of the existing draft to update"}, + "fields": {"type": "object", "description": "Fields to update in the draft's proposed content"}, + "reason": {"type": "string", "description": "Why the draft is being updated"}, + }, + "required": ["draft_id", "fields"], + }, + _update_agent_draft, + ), + "create_refined_course": AgentTool( + "create_refined_course", + "Create a new course directly in refined_submissions, or update an existing one by refined_id. Use this ONLY for brand-new courses that do not exist in the curriculum yet. For modifications to existing courses, use create_course_draft instead. Call signal_done immediately after this tool. Do not retry if this tool returns an error; report the error to the user instead.", + { + **OBJECT, + "properties": { + "refined_id": { + "type": "integer", + "description": "If provided, update this existing course. If omitted, create a new one.", + }, + "course_code": {"type": "string", "description": "e.g. UE25CS353A"}, + "course_title": {"type": "string"}, + "semester": {"type": "integer", "minimum": 1, "maximum": 8}, + "target_department": { + "type": "string", + "description": "CSE, ECE, ME, BT, EEE, AIML", + }, + "credit_category": {"type": "string", "description": "0, 2, 4, or 5"}, + "is_elective": { + "type": "boolean", + "description": "Set true for elective courses (specialization electives in semesters 5-6). Default false (core course).", + }, + "units": { + "type": "array", + "description": "Course units. Each unit must have unit_number (int), title (str), content (str, the syllabus text), and hours (int).", + "items": { + "type": "object", + "properties": { + "unit_number": {"type": "integer"}, + "title": {"type": "string"}, + "content": {"type": "string"}, + "hours": {"type": "integer"}, + }, + "required": ["unit_number", "title", "content", "hours"], + }, + }, + "objectives": {"type": "array", "items": {"type": "string"}, "description": "Learning objectives. Send as JSON array of strings."}, + "course_outcomes": {"type": "array", "items": {"type": "string"}, "description": "Measurable course outcomes. Send as JSON array of strings."}, + "text_books": {"type": "array", "items": {"type": "string"}, "description": "Textbook references with author, title, edition, publisher, year. Send as JSON array."}, + "reference_books": {"type": "array", "items": {"type": "string"}, "description": "Reference books. Send as JSON array."}, + "lab_experiments": {"type": "array", "items": {"type": "string"}, "description": "Lab experiments (only for credit_category 5). Send as JSON array."}, + "tools_languages": {"type": "string"}, + "prelude": {"type": "string"}, + "desirable_knowledge": {"type": "string"}, + }, + "required": [ + "course_code", + "course_title", + "semester", + "target_department", + "credit_category", + ], + }, + _create_refined_course, + ), + "get_curriculum_json": AgentTool( + "get_curriculum_json", + "Read template-ready JSON for the full curriculum, optionally filtered by semester.", + { + **OBJECT, + "properties": {"semester": {"type": "integer", "minimum": 1, "maximum": 8}}, + }, + _get_curriculum_json, + ), + "create_document_draft": AgentTool( + "create_document_draft", + "Create one human-reviewable document draft containing proposed changes for multiple courses. This never applies changes.", + { + **OBJECT, + "properties": { + "courses": { + "type": "array", + "items": { + "type": "object", + "properties": { + "refined_id": {"type": "integer"}, + "fields": {"type": "object"}, + }, + "required": ["refined_id", "fields"], + "additionalProperties": False, + }, + }, + "reason": {"type": "string"}, + "uploaded_document_id": {"type": "string"}, + "curriculum_version_id": {"type": "integer"}, + }, + "required": ["courses"], + }, + _create_document_draft, + ), + "get_course_draft": AgentTool( + "get_course_draft", + "Read one staged course draft and its diff summary.", + { + **OBJECT, + "properties": {"draft_id": {"type": "integer"}}, + "required": ["draft_id"], + }, + _get_course_draft, + ), + "get_document_draft": AgentTool( + "get_document_draft", + "Read one staged document draft and all linked course drafts.", + { + **OBJECT, + "properties": {"document_draft_id": {"type": "integer"}}, + "required": ["document_draft_id"], + }, + _get_document_draft, + ), + "get_preview_url": AgentTool( + "get_preview_url", + "Return the preview URL for a course, course draft, or document draft.", + { + **OBJECT, + "properties": { + "kind": { + "type": "string", + "enum": ["course", "draft", "document_draft"], + }, + "id": {"type": "integer"}, + }, + "required": ["kind", "id"], + }, + _get_preview_url, + ), + "list_courses": AgentTool( + "list_courses", + "List refined course IDs and titles, optionally filtered by semester.", + { + **OBJECT, + "properties": {"semester": {"type": "integer", "minimum": 1, "maximum": 8}}, + }, + _list_courses, + ), + "get_attachment_text": AgentTool( + "get_attachment_text", + "Read extracted text for uploaded chat attachments within a chat session.", + { + **OBJECT, + "properties": { + "session_id": {"type": "integer"}, + "attachment_ids": {"type": "array", "items": {"type": "integer"}}, + }, + "required": ["session_id", "attachment_ids"], + }, + _attachment_text, + ), + "fetch_url": AgentTool( + "fetch_url", + "Fetch a public URL and return its text content. Use to read web pages, public documents, and online resources.", + {**OBJECT, "properties": {"url": {"type": "string"}}, "required": ["url"]}, + _fetch_url, + ), + "web_search": AgentTool( + "web_search", + "Search the web and return top results with titles, snippets, and URLs. Use for finding current information, documentation, or references.", + { + **OBJECT, + "properties": { + "query": {"type": "string", "description": "Search query"}, + "num_results": { + "type": "integer", + "minimum": 1, + "maximum": 10, + "default": 5, + }, + }, + "required": ["query"], + }, + _web_search, + ), + "create_report": AgentTool( + "create_report", + "Save a generated document (report, comparison, summary, etc.) as a chat attachment accessible to the user. Use after reading source documents and generating new content. Supports markdown (.md) or PDF (.pdf) output.", + { + **OBJECT, + "properties": { + "session_id": {"type": "integer"}, + "content": { + "type": "string", + "description": "Full report/document content in markdown format", + }, + "filename": { + "type": "string", + "description": "Filename including extension, e.g. comparison-report.md or comparison-report.pdf", + }, + "format": { + "type": "string", + "enum": ["markdown", "pdf"], + "default": "markdown", + "description": "Output format: markdown or pdf", + }, + }, + "required": ["session_id", "content"], + }, + _create_report, + ), + "create_curriculum_version": AgentTool( + "create_curriculum_version", + "Create a named curriculum version snapshot (like a git commit). Use to checkpoint the curriculum state after a set of changes. Provide a descriptive name like 'feat: add CS201 lab experiments' or 'fix: correct credit hours for ECE301'.", + { + **OBJECT, + "properties": { + "name": { + "type": "string", + "description": "Descriptive version name (conventional commit style encouraged)", + }, + }, + "required": ["name"], + }, + _create_curriculum_version, + ), + "define_specialization": AgentTool( + "define_specialization", + "Create a specialization track (e.g. Machine Intelligence and Data Science) for a given semester. The letter (A, B, C...) is auto-assigned if omitted. Used to set up the elective specialization brackets.", + { + **OBJECT, + "properties": { + "semester": { + "type": "integer", + "minimum": 1, + "maximum": 8, + "description": "Semester the specialization applies to (5 or 6 for electives)", + }, + "name": { + "type": "string", + "description": "Full specialization name, e.g. 'Machine Intelligence and Data Science (MIDS)'", + }, + "letter": { + "type": "string", + "description": "Optional letter label (A, B, C). Auto-assigned if omitted.", + }, + "key": { + "type": "string", + "description": "Optional short key (SCC, MIDS, CSCS). Derived from name if omitted.", + }, + "academic_year": { + "type": "string", + "description": "Optional academic year batch, e.g. 2025-26", + }, + }, + "required": ["semester", "name"], + }, + _define_specialization, + ), + "list_specializations": AgentTool( + "list_specializations", + "List specialization track definitions, optionally filtered by semester. Use to discover which tracks exist before assigning electives.", + { + **OBJECT, + "properties": {"semester": {"type": "integer", "minimum": 1, "maximum": 8}}, + }, + _list_specializations, + ), + "assign_elective_to_tracks": AgentTool( + "assign_elective_to_tracks", + "Categorize an elective course into one or more specialization tracks by their specialization_id. Also marks the course as an elective. Use after determining (e.g. via AI analysis) which tracks the course belongs to.", + { + **OBJECT, + "properties": { + "refined_id": {"type": "integer"}, + "specialization_ids": { + "type": "array", + "items": {"type": "integer"}, + "description": "Specialization track IDs to assign the elective to", + }, + }, + "required": ["refined_id", "specialization_ids"], + }, + _assign_elective_to_tracks, + ), + "remove_elective_from_tracks": AgentTool( + "remove_elective_from_tracks", + "Remove an elective course from one or more specialization tracks.", + { + **OBJECT, + "properties": { + "refined_id": {"type": "integer"}, + "specialization_ids": { + "type": "array", + "items": {"type": "integer"}, + "description": "Specialization track IDs to remove the elective from", + }, + }, + "required": ["refined_id", "specialization_ids"], + }, + _remove_elective_from_tracks, + ), + "get_course_assignments": AgentTool( + "get_course_assignments", + "Return which specialization tracks a course currently belongs to, including the track definitions.", + { + **OBJECT, + "properties": {"refined_id": {"type": "integer"}}, + "required": ["refined_id"], + }, + _get_course_assignments, + ), + "categorize_elective": AgentTool( + "categorize_elective", + "Run guarded AI categorization for a refined elective. It assigns only existing semester tracks with high confidence; all other results require human confirmation.", + { + **OBJECT, + "properties": {"refined_id": {"type": "integer"}}, + "required": ["refined_id"], + }, + _categorize_elective, + ), + "update_deterministic_fields": AgentTool( + "update_deterministic_fields", + "Create a reviewable draft that changes deterministic/protected fields (program, lecture_hours, tutorial_hours, practical_hours, self_study, credits, course_type). This is the ONLY way to modify those fields. The resulting draft is blocked until the user explicitly approves it in the Review panel. Always confirm with the user before calling this.", + { + **OBJECT, + "properties": { + "refined_id": {"type": "integer"}, + "fields": { + "type": "object", + "description": 'Protected fields to change, e.g. {"credits": 5, "lecture_hours": 4}', + }, + "reason": {"type": "string"}, + }, + "required": ["refined_id", "fields"], + }, + _update_deterministic_fields, + ), + "signal_done": AgentTool( + "signal_done", + "Signal that the agent has completed the user's request. Provide a concise summary of what was accomplished. This ends the agent's turn.", + { + **OBJECT, + "properties": { + "summary": { + "type": "string", + "description": "Brief summary of what was done, e.g. 'Created draft for CS201 adding Unit 5 on Graph Algorithms'", + }, + }, + "required": ["summary"], + }, + _signal_done, + ), + "ask_user": AgentTool( + "ask_user", + "Ask the user a clarifying question before proceeding. Call this when you need more information (credit category, semester, course code, elective vs core, etc.). The stream ends after this tool, and the user will respond in their next message.", + { + **OBJECT, + "properties": { + "question": { + "type": "string", + "description": "The question to ask the user", + }, + }, + "required": ["question"], + }, + _ask_user, + ), + "create_spreadsheet": AgentTool( + "create_spreadsheet", + "Generate a CSV or Excel file from structured row data and save as a downloadable chat attachment. Provide column names and row objects.", + { + **OBJECT, + "properties": { + "session_id": {"type": "integer"}, + "columns": { + "type": "array", + "items": {"type": "string"}, + "description": "Column names in display order", + }, + "rows": { + "type": "array", + "items": {"type": "object"}, + "description": "Array of row objects, keys matching column names", + }, + "filename": { + "type": "string", + "description": "Filename including extension, e.g. semester-3-courses.xlsx", + }, + "format": {"type": "string", "enum": ["csv", "xlsx"], "default": "csv"}, + }, + "required": ["session_id", "columns", "rows"], + }, + _create_spreadsheet, + ), + "get_version": AgentTool( + "get_version", + "Load a curriculum version snapshot with its metadata and course list. Use to inspect what a version contains before comparing or restoring.", + { + **OBJECT, + "properties": { + "version_id": {"type": "integer", "description": "Version snapshot ID"}, + }, + "required": ["version_id"], + }, + _get_version, + ), + "diff_versions": AgentTool( + "diff_versions", + "Compare two curriculum version snapshots. Returns added, removed, and changed courses with per-course change percentages.", + { + **OBJECT, + "properties": { + "version_a": { + "type": "integer", + "description": "First version ID (base)", + }, + "version_b": { + "type": "integer", + "description": "Second version ID (target)", + }, + }, + "required": ["version_a", "version_b"], + }, + _diff_versions, + ), + "get_curriculum_stats": AgentTool( + "get_curriculum_stats", + "Compute aggregate curriculum statistics: total courses, credits per semester, course type distribution, visible vs hidden counts. Optionally filter to one semester.", + { + **OBJECT, + "properties": { + "semester": { + "type": "integer", + "minimum": 1, + "maximum": 8, + "description": "Optional: stats for one semester only", + }, + }, + }, + _get_curriculum_stats, + ), + "batch_read_courses": AgentTool( + "batch_read_courses", + "Read specific fields from multiple courses in one call. More efficient than calling get_course_fields multiple times. Returns an array of course field subsets.", + { + **OBJECT, + "properties": { + "refined_ids": { + "type": "array", + "items": {"type": "integer"}, + "description": "List of course refined IDs to read", + }, + "fields": { + "type": "array", + "items": {"type": "string"}, + "description": "Field names to read for each course", + }, + }, + "required": ["refined_ids", "fields"], + }, + _batch_read_courses, + ), +} diff --git a/backend/app/services/attachments.py b/backend/app/services/attachments.py new file mode 100644 index 0000000000000000000000000000000000000000..892f51b7b77e94a980af95344581d6e606051540 --- /dev/null +++ b/backend/app/services/attachments.py @@ -0,0 +1,91 @@ +import re +import subprocess +import tempfile +import zipfile +from pathlib import Path +from xml.etree import ElementTree + +MAX_FILE_BYTES = 8 * 1024 * 1024 +MAX_TEXT_CHARS = 200000 +SPACE = re.compile(r"[ \t]+") +BLANKS = re.compile(r"\n{3,}") + + +def extract_text(filename: str, content_type: str, data: bytes) -> tuple[str, str, str]: + if len(data) > MAX_FILE_BYTES: + return "", "failed", "File is larger than 8 MB" + suffix = Path(filename or "").suffix.lower() + try: + if suffix == ".pdf" or content_type == "application/pdf": + return _clean(_pdf_text(data)), "ready", "" + if suffix == ".docx": + return _clean(_docx_text(data)), "ready", "" + if suffix == ".xlsx": + return _clean(_xlsx_text(data)), "ready", "" + if suffix in {".txt", ".md", ".csv"} or content_type.startswith("text/"): + return _clean(_decode(data)), "ready", "" + if content_type.startswith("image/"): + return "", "unsupported", "Image OCR is not enabled yet" + return "", "unsupported", "File type is not supported yet" + except Exception as exc: + return "", "failed", str(exc) + + +def _pdf_text(data: bytes) -> str: + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "upload.pdf" + path.write_bytes(data) + result = subprocess.run( + ["pdftotext", "-layout", str(path), "-"], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + return result.stdout + + +def _docx_text(data: bytes) -> str: + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "upload.docx" + path.write_bytes(data) + with zipfile.ZipFile(path) as docx: + xml = docx.read("word/document.xml") + root = ElementTree.fromstring(xml) + ns = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + lines = [] + for paragraph in root.iter(f"{ns}p"): + text = "".join(node.text or "" for node in paragraph.iter(f"{ns}t")).strip() + if text: + lines.append(text) + return "\n".join(lines) + + +def _xlsx_text(data: bytes) -> str: + import openpyxl + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "upload.xlsx" + path.write_bytes(data) + wb = openpyxl.load_workbook(path, read_only=True, data_only=True) + rows = [] + for sheet in wb: + for row in sheet.iter_rows(values_only=True): + line = "\t".join(str(c or "") for c in row) + if line.strip(): + rows.append(line) + return "\n".join(rows) + + +def _decode(data: bytes) -> str: + for encoding in ("utf-8", "utf-16", "latin-1"): + try: + return data.decode(encoding) + except UnicodeDecodeError: + continue + return data.decode("utf-8", errors="ignore") + + +def _clean(text: str) -> str: + text = SPACE.sub(" ", text.replace("\r\n", "\n").replace("\r", "\n")) + text = BLANKS.sub("\n\n", text).strip() + return text[:MAX_TEXT_CHARS] diff --git a/backend/app/services/books.py b/backend/app/services/books.py new file mode 100644 index 0000000000000000000000000000000000000000..3e92d310172500b5d546a2d2b299bb3dfe5451aa --- /dev/null +++ b/backend/app/services/books.py @@ -0,0 +1,105 @@ +import re + +WORD = re.compile(r"\w+") +BOOK_STOP = re.compile( + r"\bCourse\b[\s\S]{0,160}\bOutcome\b|" + r"\b(Course\s+Objectives?|Assignment\s*/|Laboratory|Recommended\s+Materials)\b", + re.IGNORECASE, +) +BOOK_LABEL = re.compile( + r"(?im)^\s*(?:Text\s*)?Book\(s\):\s*|" + r"^\s*Text\s+Book\(s\):\s*|" + r"^\s*Reference\s*(?:Book\(s\):)?\s*" +) +BOOK_NUMBER = re.compile(r"(?:^|\n|\s{2,}|(?<=\.)\s+)\d+\s*[:.)]\s*") +ORDINAL_LINE = re.compile(r"(?im)^\s*(st|nd|rd|th)\s*$") +PAGE_NOISE = re.compile( + r"P\.?\s*E\.?\s*S\.?\s*University|" + r"Curriculum|" + r"\s*:-\s*[A-Za-z]*\s*\d{4}\s*-\s*\d{4}\b|" + r"\b\d+\s*\|\s*Page\b", + re.IGNORECASE, +) +TEXT_BOOK_LINE = re.compile(r"\bText\s+Book\(s\):", re.IGNORECASE) +REFERENCE_LINE = re.compile(r"^\s*Reference\b", re.IGNORECASE) +SECTION_END_LINE = re.compile(r"^\s*(Course\s+Outcome|Assignment\s*/|Course\s+Objectives?|Laboratory)\b", re.IGNORECASE) + + +def _words(text: str) -> int: + return len(WORD.findall(text)) + + +def _flatten(values) -> str: + parts = [] + for value in values: + if not value: + continue + if isinstance(value, list): + parts.extend(str(item) for item in value if str(item).strip()) + else: + parts.append(str(value)) + text = "\n".join(parts) + match = BOOK_STOP.search(text) + if match: + text = text[: match.start()] + text = ORDINAL_LINE.sub("", text) + text = PAGE_NOISE.sub("\n", text) + return BOOK_LABEL.sub("", text) + + +def _clean(value: str) -> str: + value = BOOK_LABEL.sub("", value) + value = re.sub(r"\bBook\(s\):?\s*", "", value, flags=re.IGNORECASE) + value = re.sub(r"-\s+", "-", value) + return re.sub(r"\s+", " ", value).strip(" :") + + +def parse_books(*values) -> list[str]: + text = _flatten(values) + if not text.strip(): + return [] + + numbered = [part for part in BOOK_NUMBER.split(text) if part.strip()] + if numbered and (len(numbered) > 1 or BOOK_NUMBER.search(text)): + parts = numbered + else: + parts = [line for line in text.splitlines() if line.strip()] + + books = [] + seen = set() + for part in parts: + item = _clean(part) + if not item: + continue + if books and (item.startswith("(") or (_words(item) <= 3 and not re.search(r"\b\d{4}\b", item))): + books[-1] = f"{books[-1]} {item}".strip() + continue + key = re.sub(r"[^a-z0-9]+", " ", item.lower()).strip() + if key not in seen: + books.append(item) + seen.add(key) + return books + + +def raw_book_section(raw_content: str, kind: str) -> str: + lines = (raw_content or "").splitlines() + start = None + for index, line in enumerate(lines): + if kind == "text" and TEXT_BOOK_LINE.search(line): + start = index + break + if kind == "reference" and REFERENCE_LINE.search(line): + start = index + break + if start is None: + return "" + + end = len(lines) + for index in range(start + 1, len(lines)): + if kind == "text" and REFERENCE_LINE.search(lines[index]): + end = index + break + if SECTION_END_LINE.search(lines[index]): + end = index + break + return "\n".join(lines[start:end]) diff --git a/backend/app/services/curriculum.py b/backend/app/services/curriculum.py new file mode 100644 index 0000000000000000000000000000000000000000..3660ee60cd8fc80bade6f7908e74923f03fd7df9 --- /dev/null +++ b/backend/app/services/curriculum.py @@ -0,0 +1,225 @@ +from os import environ + +from app import cache +from app.preview import build_course_preview +from app.services.diffing import diff_course, merge_fields, validate_draft +from app.supabase import first_row, supabase + +DEFAULT_CURRICULUM_YEAR = environ.get("CURRICULUM_YEAR", "").strip() + + +def invalidate_curriculum_cache(): + """Invalidate all cached curriculum PDF/HTML after data changes.""" + cache.invalidate("full_pdf:") + cache.invalidate("full_html:") + cache.invalidate("sem_pdf:") + cache.invalidate("course:") + cache.invalidate("course_html:") + cache.invalidate("course_pdf:") + cache.invalidate("sem_courses:") + cache.invalidate("courses_list") + cache.invalidate("pending_courses") + cache.invalidate("all_course_ids") + cache.invalidate("ver_preview:") +SOURCE_ORDER = { + "UE25CS151A": 1, + "UE25CS151B": 1, + "UE24CS251A": 1, + "UE24CS252A": 2, + "UE24MA242A": 3, + "UE24CS241A": 4, + "UE24CS243A": 5, + "UZ24UZ221A": 6, + "UE25MA201A*": 7, + "UE24CS251B": 1, + "UE24CS252B": 2, + "UE24CS241B": 3, + "UE24CS242B": 4, + "UE24MA241B": 5, + "UZ24UZ221B": 6, + "UE25MA201B*": 7, + "UE23CS351A": 1, + "UE23CS352A": 2, + "UE23CS341A": 3, + "UE23CS342AAX": 4, + "UE23CS343ABX": 5, + "UE23CS320A": 6, + "UE23CS351B": 1, + "UE23CS352B": 2, + "UE23CS341B": 3, + "UE23CS342BAX": 4, + "UE23CS343BBX": 5, + "UE23CS320B": 6, + "UE22CS441A": 1, + "UZ22UZ422A": 2, + "UE22AM421AXX": 3, + "UE22CS421B": 1, + "UE22CS461XB": 2, +} + +REFINED_FIELDS = { + "semester", + "course_code", + "course_title", + "program", + "lecture_hours", + "tutorial_hours", + "practical_hours", + "self_study", + "credits", + "course_type", + "is_elective", + "tools_languages", + "desirable_knowledge", + "prelude", + "objectives", + "course_outcomes", + "units", + "lab_experiments", + "text_books", + "reference_books", + "status", +} + + +def attach_submissions(rows: list[dict]) -> list[dict]: + ids = [row["submission_id"] for row in rows if row.get("submission_id")] + if not ids: + return rows + cache_key = f"attach:{','.join(str(i) for i in sorted(ids))}" + cached = cache.get(cache_key) + if cached is not None: + for row in rows: + row["_submission"] = cached.get(row.get("submission_id"), {}) + return rows + submissions = supabase.table("submissions").select("*").in_("id", ids).execute().data + by_id = {row["id"]: row for row in submissions} + cache.put(cache_key, by_id, ttl=300) + for row in rows: + row["_submission"] = by_id.get(row.get("submission_id"), {}) + return rows + + +def ordered_courses(rows: list[dict]) -> list[dict]: + rows = attach_submissions(rows) + rows.sort(key=course_sort_key) + return [build_course_preview(row) for row in rows] + + +def course_credits(row: dict) -> int: + value = row.get("credits") + if value not in (None, ""): + return int(value) + category = str((row.get("_submission") or {}).get("credit_category") or "").strip() + if category.isdigit(): + return int(category) + return 0 + + +def course_sort_key(row: dict) -> tuple[int, int, int, int]: + semester = int(row.get("semester") or 0) + code = str(row.get("course_code") or "").replace(" ", "").upper() + order = SOURCE_ORDER.get(code) + if order is None and semester == 5: + order = elective_order(code, "AA", "AB") + if order is None and semester == 6: + order = elective_order(code, "BA", "BB") + return semester, -course_credits(row), order or 900, int(row.get("id") or 0) + + +def elective_order(code: str, first_group: str, second_group: str) -> int | None: + for offset, group in ((100, first_group), (200, second_group)): + if group in code: + suffix = code.rsplit(group, 1)[-1].rstrip("X") + return offset + int(suffix) if suffix.isdigit() else offset + return None + + +def create_version_snapshot(name: str) -> dict | None: + rows = supabase.table("refined_submissions").select("*").in_("status", ["refined"]).execute().data + rows = attach_submissions(rows) + courses = [{"refined_id": row["id"], "course_json": build_course_preview(row)} for row in rows] + program = courses[0]["course_json"].get("program") if courses else "" + result = ( + supabase.table("curriculum_versions") + .insert({"name": name, "program": program, "academic_year": selected_curriculum_year(), "status": "draft"}) + .execute() + ) + version = (result.data or [None])[0] + if not version: + return None + if courses: + records = [{**course, "curriculum_version_id": version["id"]} for course in courses] + supabase.table("finalized_submissions").insert(records).execute() + return version + + +def selected_curriculum_year(override: str | None = None) -> str: + if override and override.strip(): + return override.strip() + return DEFAULT_CURRICULUM_YEAR + + +def refined_course(refined_id: int) -> dict: + cache_key = f"course:{refined_id}" + cached = cache.get(cache_key) + if cached is not None: + return cached + row = first_row(supabase.table("refined_submissions").select("*").eq("id", refined_id)) + if not row: + raise LookupError("Refined submission not found") + result = build_course_preview(attach_submissions([row])[0]) + cache.put(cache_key, result, ttl=300) + return result + + +def update_refined_fields(refined_id: int, fields: dict) -> dict | None: + update = {key: fields[key] for key in REFINED_FIELDS if key in fields} + for key in ("semester", "lecture_hours", "tutorial_hours", "practical_hours", "self_study", "credits"): + if key in update: + try: + update[key] = int(update[key] or 0) + except (TypeError, ValueError) as exc: + raise ValueError(f"Invalid value for {key}: {update[key]!r}. Must be a number.") from exc + is_elective = update.get("is_elective") + if is_elective is None: + existing = first_row(supabase.table("refined_submissions").select("is_elective").eq("id", refined_id)) + is_elective = existing.get("is_elective") if existing else False + if is_elective: + update["course_type"] = "Elective" + result = supabase.table("refined_submissions").update(update).eq("id", refined_id).execute() + invalidate_curriculum_cache() + return result.data[0] if result.data else None + + +def draft_record(refined_id: int, fields: dict, reason: str = "", document_draft_id: int | None = None) -> dict: + base = refined_course(refined_id) + proposed = merge_fields(base, fields) + summary = diff_course(base, proposed) + issues = validate_draft(base, proposed) + summary["validation_issues"] = issues + return { + "refined_id": refined_id, + "document_draft_id": document_draft_id, + "base_refined_json": base, + "proposed_json": proposed, + "json_patch": summary.pop("json_patch"), + "diff_summary": summary, + "change_reason": reason.strip(), + "status": "blocked" if issues else "proposed", + } + + +def load_agent_draft(draft_id: int) -> dict: + draft = first_row(supabase.table("agent_drafts").select("*").eq("id", draft_id)) + if not draft: + raise LookupError("Agent draft not found") + return draft + + +def load_document_draft(document_draft_id: int) -> dict: + document = first_row(supabase.table("agent_document_drafts").select("*").eq("id", document_draft_id)) + if not document: + raise LookupError("Document draft not found") + drafts = supabase.table("agent_drafts").select("*").eq("document_draft_id", document_draft_id).order("id").execute().data + return {"document_draft": document, "drafts": drafts} diff --git a/backend/app/services/deterministic.py b/backend/app/services/deterministic.py new file mode 100644 index 0000000000000000000000000000000000000000..9b33cf86a45a20e50304031e911a79977f13eba0 --- /dev/null +++ b/backend/app/services/deterministic.py @@ -0,0 +1,43 @@ +_HOURS_MAP = { + "5": {"lecture_hours": 4, "tutorial_hours": 0, "practical_hours": 2, "self_study": 5, "credits": 5}, + "4": {"lecture_hours": 4, "tutorial_hours": 0, "practical_hours": 0, "self_study": 4, "credits": 4}, + "2": {"lecture_hours": 2, "tutorial_hours": 0, "practical_hours": 0, "self_study": 2, "credits": 2}, + "0": {"lecture_hours": 0, "tutorial_hours": 0, "practical_hours": 0, "self_study": 0, "credits": 0}, +} + +_PROGRAM_MAP = { + "CSE": "B. TECH", + "AIML": "B. TECH", + "ECE": "B. TECH", + "ME": "B. TECH", + "EEE": "B. TECH", + "BT": "B. TECH", +} + +_COURSE_TYPE_MAP = { + "0": "Foundation Course", + "5": "Core Course-Lab Integrated", + "4": "Core Course", + "2": "Core Theory", +} + + +def compute_hours(cat: str) -> dict: + result = _HOURS_MAP.get(cat) + if result is None: + raise ValueError(f"Unknown credit_category: {cat!r}. Expected one of: {sorted(_HOURS_MAP)}") + return result + + +def compute_program(dept: str) -> str: + result = _PROGRAM_MAP.get(dept) + if result is None: + raise ValueError(f"Unknown department: {dept!r}. Expected one of: {sorted(_PROGRAM_MAP)}") + return result + + +def compute_course_type(cat: str) -> str: + result = _COURSE_TYPE_MAP.get(cat) + if result is None: + raise ValueError(f"Unknown credit_category: {cat!r}. Expected one of: {sorted(_COURSE_TYPE_MAP)}") + return result diff --git a/backend/app/services/diffing.py b/backend/app/services/diffing.py new file mode 100644 index 0000000000000000000000000000000000000000..9282c47b749913eb14134485aa703aa16c0f0c50 --- /dev/null +++ b/backend/app/services/diffing.py @@ -0,0 +1,241 @@ +import difflib +import json +import re +from copy import deepcopy +from typing import Any + +PROTECTED_FIELDS = { + "program", + "lecture_hours", + "tutorial_hours", + "practical_hours", + "self_study", + "credits", + "course_type", +} + +TOPIC_SPLIT = re.compile(r"[,;\n]+") +SPACE = re.compile(r"\s+") + + +def stable_json(value: Any) -> str: + return json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + + +def build_patch(old: Any, new: Any, path: str = "") -> list[dict]: + if isinstance(old, dict) and isinstance(new, dict): + patch = [] + old_keys = set(old) + new_keys = set(new) + for key in sorted(old_keys - new_keys): + patch.append({"op": "remove", "path": f"{path}/{_escape(key)}"}) + for key in sorted(new_keys - old_keys): + patch.append({"op": "add", "path": f"{path}/{_escape(key)}", "value": new[key]}) + for key in sorted(old_keys & new_keys): + patch.extend(build_patch(old[key], new[key], f"{path}/{_escape(key)}")) + return patch + + if old != new: + return [{"op": "replace", "path": path or "/", "value": new}] + return [] + + +def apply_patch(value: Any, patch: list[dict]) -> Any: + result = deepcopy(value) + for op in patch: + name = op.get("op") + path = op.get("path") + if name not in {"add", "remove", "replace"} or not isinstance(path, str): + raise ValueError("Unsupported patch operation") + parent, key = _resolve_parent(result, path) + if isinstance(parent, list): + index = int(key) + if name == "remove": + parent.pop(index) + elif name == "replace": + parent[index] = op.get("value") + else: + parent.insert(index, op.get("value")) + continue + if name == "remove": + parent.pop(key, None) + else: + parent[key] = op.get("value") + return result + + +def diff_course(old: dict, new: dict) -> dict: + old_text = stable_json(old) + new_text = stable_json(new) + patch = build_patch(old, new) + syllabus_old = _syllabus_text(old) + syllabus_new = _syllabus_text(new) + topics_old = _topics(syllabus_old) + topics_new = _topics(syllabus_new) + + def _is_protected(field: str) -> bool: + if field == "course_type" and new.get("is_elective"): + return False + return field in PROTECTED_FIELDS + + return { + "json_patch": patch, + "patch_operations": len(patch), + "change_percent": _change_percent(old_text, new_text), + "syllabus_change_percent": _change_percent(syllabus_old, syllabus_new), + "topics_added": sorted(topics_new - topics_old), + "topics_removed": sorted(topics_old - topics_new), + "protected_changes": sorted(field for field in PROTECTED_FIELDS if _is_protected(field) and _field_value(old, field) != _field_value(new, field)), + "unified_diff": "\n".join( + difflib.unified_diff( + old_text.splitlines(), + new_text.splitlines(), + fromfile="current", + tofile="proposed", + lineterm="", + ) + ), + } + + +def validate_draft(old: dict, new: dict) -> list[str]: + issues = [] + for field in PROTECTED_FIELDS: + if field == "course_type" and new.get("is_elective"): + continue + if _field_value(old, field) != _field_value(new, field): + issues.append(f"{field} is deterministic and cannot be changed by an agent draft") + return issues + + +def merge_fields(base: dict, fields: dict) -> dict: + merged = deepcopy(base) + for key, value in fields.items(): + if key in PROTECTED_FIELDS and str(base.get(key) or "") == str(value or ""): + value = base.get(key) + merged[key] = value + if merged.get("is_elective"): + merged["course_type"] = "Elective" + return merged + + +def _field_value(value: dict, field: str) -> str: + return str(value.get(field) or "") + + +def _escape(value: str) -> str: + return value.replace("~", "~0").replace("/", "~1") + + +def _unescape(value: str) -> str: + return value.replace("~1", "/").replace("~0", "~") + + +def _resolve_parent(value: Any, path: str) -> tuple[Any, str]: + if path in {"", "/"}: + raise ValueError("Root patch operations are not supported") + parts = [_unescape(part) for part in path.strip("/").split("/")] + current = value + for part in parts[:-1]: + current = current[int(part)] if isinstance(current, list) else current[part] + return current, parts[-1] + + +def _change_percent(old: str, new: str) -> float: + if not old and not new: + return 0.0 + ratio = difflib.SequenceMatcher(None, old, new).ratio() + return round((1 - ratio) * 100, 2) + + +def _syllabus_text(value: dict) -> str: + units = value.get("units") or [] + if not isinstance(units, list): + return "" + parts = [] + for unit in units: + if isinstance(unit, dict): + parts.append(str(unit.get("title") or "")) + parts.append(str(unit.get("content") or "")) + return SPACE.sub(" ", " ".join(parts)).strip() + + +def _topics(text: str) -> set[str]: + topics = set() + for item in TOPIC_SPLIT.split(text): + topic = SPACE.sub(" ", item).strip(" .:-").lower() + if len(topic) >= 4: + topics.add(topic) + return topics + + +def diff_text_field(old: str, new: str) -> dict | None: + if old == new: + return None + return {"kind": "text", "old": old or "", "new": new or ""} + + +def diff_list_field(old: list, new: list) -> dict | None: + if old == new: + return None + old_set = set(old) + new_set = set(new) + return { + "kind": "list", + "removed": sorted(old_set - new_set), + "added": sorted(new_set - old_set), + "unchanged": sorted(old_set & new_set), + } + + +def diff_units_field(old_units: list, new_units: list) -> dict | None: + if old_units == new_units: + return None + old_by_title = {u.get("title", ""): u for u in old_units if isinstance(u, dict)} + new_by_title = {u.get("title", ""): u for u in new_units if isinstance(u, dict)} + all_titles = sorted(set(old_by_title.keys()) | set(new_by_title.keys())) + units_diff = [] + for title in all_titles: + old_u = old_by_title.get(title) + new_u = new_by_title.get(title) + if old_u and not new_u: + units_diff.append({"kind": "removed", "unit": old_u}) + elif new_u and not old_u: + units_diff.append({"kind": "added", "unit": new_u}) + else: + unit_changes = {} + for field in ("title", "content", "hours"): + ov = old_u.get(field, "") + nv = new_u.get(field, "") + if ov != nv: + unit_changes[field] = {"old": ov, "new": nv} + if unit_changes: + units_diff.append({"kind": "changed", "unit": new_u, "changes": unit_changes}) + else: + units_diff.append({"kind": "unchanged", "unit": new_u}) + return {"kind": "units", "units": units_diff} + + +def build_course_diff(base: dict, proposed: dict) -> dict: + diff = { + "course_title": diff_text_field(base.get("course_title", ""), proposed.get("course_title", "")), + "course_code": diff_text_field(base.get("course_code", ""), proposed.get("course_code", "")), + "program": diff_text_field(base.get("program", ""), proposed.get("program", "")), + "lecture_hours": diff_text_field(str(base.get("lecture_hours", "")), str(proposed.get("lecture_hours", ""))), + "tutorial_hours": diff_text_field(str(base.get("tutorial_hours", "")), str(proposed.get("tutorial_hours", ""))), + "practical_hours": diff_text_field(str(base.get("practical_hours", "")), str(proposed.get("practical_hours", ""))), + "self_study": diff_text_field(str(base.get("self_study", "")), str(proposed.get("self_study", ""))), + "credits": diff_text_field(str(base.get("credits", "")), str(proposed.get("credits", ""))), + "course_type": diff_text_field(base.get("course_type", ""), proposed.get("course_type", "")), + "semester": diff_text_field(str(base.get("semester", "")), str(proposed.get("semester", ""))), + "tools_languages": diff_text_field(base.get("tools_languages", ""), proposed.get("tools_languages", "")), + "desirable_knowledge": diff_text_field(base.get("desirable_knowledge", ""), proposed.get("desirable_knowledge", "")), + "prelude": diff_text_field(base.get("prelude", ""), proposed.get("prelude", "")), + "objectives": diff_list_field(base.get("objectives") or [], proposed.get("objectives") or []), + "course_outcomes": diff_list_field(base.get("course_outcomes") or [], proposed.get("course_outcomes") or []), + "units": diff_units_field(base.get("units") or [], proposed.get("units") or []), + "lab_experiments": diff_list_field(base.get("lab_experiments") or [], proposed.get("lab_experiments") or []), + "text_books": diff_list_field(base.get("text_books") or [], proposed.get("text_books") or []), + "reference_books": diff_list_field(base.get("reference_books") or [], proposed.get("reference_books") or []), + } + return diff diff --git a/backend/app/services/elective_categorization.py b/backend/app/services/elective_categorization.py new file mode 100644 index 0000000000000000000000000000000000000000..d37f04433485a120bd8dd3f9c0e7e452e702fa85 --- /dev/null +++ b/backend/app/services/elective_categorization.py @@ -0,0 +1,202 @@ +"""Guarded AI categorization for newly refined elective courses.""" + +import logging + +from app.services.openrouter import call as llm +from app.supabase import first_row, supabase + +logger = logging.getLogger(__name__) + +MIN_CONFIDENCE = 0.80 +ELECTIVE_MARKERS = ("AA", "AB", "BA", "BB") +SYSTEM_PROMPT = """You categorize PES University elective courses into existing specialization tracks. +Return only valid JSON in this exact shape: +{"confidence": 0.0, "assignments": [{"specialization_id": 1, "confidence": 0.0, "reasoning": "Short evidence-based reason."}]} +Use only IDs supplied in the prompt. Never invent a track or ID. Assign only clear matches. +For no clear match, return an empty assignments list and confidence below 0.80. +""" + + +def is_elective_course(course: dict) -> bool: + """Mirror the legacy elective-code grouping rule for semesters 5 and 6.""" + code = str(course.get("course_code") or "").replace(" ", "").upper() + return int(course.get("semester") or 0) in (5, 6) and any( + marker in code for marker in ELECTIVE_MARKERS + ) + + +def _course_content(course: dict) -> str: + """Extract a flat text representation of course content for the LLM prompt.""" + units = course.get("units") or [] + unit_text = [] + if isinstance(units, list): + for unit in units: + if isinstance(unit, dict): + unit_text.append( + f"{unit.get('title', '')}: {unit.get('content', '')}".strip() + ) + else: + unit_text.append(str(unit)) + parts = [ + course.get("course_title"), + course.get("prelude"), + course.get("objectives"), + course.get("course_outcomes"), + *unit_text, + course.get("tools_languages"), + ] + return "\n".join(str(p) for p in parts if p).strip()[:18000] + + +def _confirmation(reason: str, **extra) -> dict: + return {"assigned": False, "needs_human_confirmation": True, "reason": reason, **extra} + + +def _fetch_tracks(semester: int) -> list[dict]: + """Return specialization definitions for the given semester.""" + return ( + supabase.table("specialization_definitions") + .select("id,semester,letter,name,key,academic_year") + .eq("semester", semester) + .order("letter") + .execute() + .data + ) + + +def _validate_assignments(assignments: list, valid_ids: set[int]) -> dict | None: + """Validate each assignment entry; returns an error confirmation or None on success.""" + seen: set[int] = set() + for assignment in assignments: + try: + spec_id = int(assignment["specialization_id"]) + conf = float(assignment["confidence"]) + reasoning = str(assignment["reasoning"]).strip() + except (KeyError, TypeError, ValueError): + return _confirmation("invalid_model_assignment") + if spec_id not in valid_ids: + return _confirmation( + "unknown_specialization_id", specialization_id=spec_id + ) + if conf < MIN_CONFIDENCE or not reasoning: + return _confirmation( + "low_assignment_confidence", specialization_id=spec_id + ) + if spec_id not in seen: + seen.add(spec_id) + return None + + +def _persist_assignments(refined_id: int, accepted: list[dict]) -> int: + """Insert accepted assignments into the DB; returns the number created.""" + created = 0 + for assignment in accepted: + spec_id = assignment["specialization_id"] + exists = ( + supabase.table("course_specialization_assignments") + .select("id") + .eq("refined_id", refined_id) + .eq("specialization_id", spec_id) + .execute() + .data + ) + if not exists: + supabase.table("course_specialization_assignments").insert( + {"refined_id": refined_id, "specialization_id": spec_id} + ).execute() + created += 1 + return created + + +def _parse_llm_response(result) -> tuple[float, list[dict]] | None: + """Parse and validate the LLM response; returns (confidence, assignments) or None.""" + if not isinstance(result, dict): + return None + try: + confidence = float(result.get("confidence")) + except (TypeError, ValueError): + return None + assignments = result.get("assignments") + if not isinstance(assignments, list) or not assignments: + return None + return confidence, assignments + + +def categorize_refined_elective(refined_id: int) -> dict: + """Assign only existing specialization tracks with validated high-confidence output.""" + course = first_row( + supabase.table("refined_submissions").select("*").eq("id", refined_id) + ) + if not course: + raise LookupError("Refined submission not found") + if not course.get("is_elective") and not is_elective_course(course): + return { + "assigned": False, + "needs_human_confirmation": False, + "reason": "not_an_elective", + } + if not course.get("is_elective"): + supabase.table("refined_submissions").update({"is_elective": True}).eq( + "id", refined_id + ).execute() + + tracks = _fetch_tracks(int(course["semester"])) + if not tracks: + return _confirmation("no_specializations_for_semester", refined_id=refined_id) + + try: + result = llm( + SYSTEM_PROMPT, + f"""Course ID: {refined_id} +Course code: {course.get("course_code") or ""} +Course content: +{_course_content(course)} +Available specialization tracks: {tracks}""", + ) + except Exception: + logger.exception( + "Elective categorization model call failed for refined_id=%s", refined_id + ) + return _confirmation("model_error", refined_id=refined_id) + + parsed = _parse_llm_response(result) + if parsed is None: + return _confirmation("invalid_model_response", refined_id=refined_id) + + confidence, assignments = parsed + if confidence < MIN_CONFIDENCE: + return _confirmation( + "low_confidence_or_no_match", refined_id=refined_id, confidence=confidence + ) + + valid_ids = {int(track["id"]) for track in tracks} + validation_error = _validate_assignments(assignments, valid_ids) + if validation_error: + validation_error["refined_id"] = refined_id + return validation_error + + seen: set[int] = set() + accepted: list[dict] = [] + for assignment in assignments: + spec_id = int(assignment["specialization_id"]) + if spec_id not in seen: + seen.add(spec_id) + accepted.append( + { + "specialization_id": spec_id, + "confidence": float(assignment["confidence"]), + "reasoning": str(assignment["reasoning"]).strip(), + } + ) + + created = _persist_assignments(refined_id, accepted) + logger.info( + "Elective categorization refined_id=%s assignments=%s", refined_id, accepted + ) + return { + "assigned": True, + "needs_human_confirmation": False, + "refined_id": refined_id, + "assignments_created": created, + "assignments": accepted, + } diff --git a/backend/app/services/errors.py b/backend/app/services/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..e74f8b53aec81b2949b2e2202fa8694a4c64029a --- /dev/null +++ b/backend/app/services/errors.py @@ -0,0 +1,16 @@ +import logging + +import sentry_sdk +from fastapi import HTTPException +from postgrest.exceptions import APIError + +logger = logging.getLogger(__name__) + + +def database_http_exception(exc: APIError) -> HTTPException: + message = str(exc) + if "schema cache" in message and "Could not find the table" in message: + return HTTPException(status_code=503, detail="Required database tables are missing. Run docs/schema.sql in Supabase.") + logger.error("Supabase APIError: %s", message) + sentry_sdk.capture_exception(exc) + return HTTPException(status_code=500, detail=f"Database request failed: {message[:200]}") diff --git a/backend/app/services/openrouter.py b/backend/app/services/openrouter.py new file mode 100644 index 0000000000000000000000000000000000000000..b91851ddda6cc3b0f98180c119e3b047e8e25cb8 --- /dev/null +++ b/backend/app/services/openrouter.py @@ -0,0 +1,475 @@ +import json +import logging +import os +import random +import time + +import httpx +from dotenv import load_dotenv +from httpx import Client, HTTPError, HTTPStatusError + +load_dotenv("../.env") + +URL = os.environ["OPENROUTER_URL"].strip() +KEY = os.environ["OPENROUTER_API_KEY"].strip() +MODEL = os.environ["OPENROUTER_MODEL"].strip() +FALLBACK_MODEL = os.environ.get("OPENROUTER_FALLBACK_MODEL", "").strip() or None +logger = logging.getLogger(__name__) + +_context_length: int | None = None +_last_fetch_attempt: float = 0.0 + + +def fetch_context_length() -> int: + global _context_length, _last_fetch_attempt + if _context_length is not None: + return _context_length + now = time.time() + if now - _last_fetch_attempt < 60: + return _context_length or 128000 + _last_fetch_attempt = now + api_base = URL.rsplit("/chat/completions", 1)[0] + try: + with Client(timeout=15) as client: + resp = client.get(f"{api_base}/model/{MODEL}", headers=_headers()) + resp.raise_for_status() + data = resp.json().get("data") or {} + _context_length = int(data.get("context_length") or 128000) + logger.info("Model %s context_length=%d", MODEL, _context_length) + return _context_length + except HTTPStatusError as exc: + if exc.response.status_code == 404: + logger.debug("Model info endpoint not available for %s (provider does not support it)", MODEL) + else: + logger.warning("Failed to fetch context_length for %s: %s", MODEL, exc) + return 128000 + except Exception as exc: + logger.warning("Failed to fetch context_length for %s: %s", MODEL, exc) + return 128000 + + +def context_length() -> int: + return _context_length or 128000 + + +_TOOL_LABELS = { + "get_course_codes": "Looking up courses", + "get_current_course_json": "Reading course data", + "get_course_syllabus": "Reading syllabus", + "get_course_textbooks": "Reading textbooks", + "get_course_deterministic": "Reading course properties", + "get_course_lab": "Reading lab details", + "get_course_fields": "Reading course fields", + "batch_read_courses": "Reading courses", + "get_curriculum_json": "Loading curriculum", + "get_curriculum_stats": "Computing statistics", + "create_course_draft": "Creating draft", + "create_refined_course": "Creating course", + "create_document_draft": "Creating document draft", + "create_report": "Generating report", + "create_spreadsheet": "Generating spreadsheet", + "diff_course_json": "Comparing courses", + "diff_versions": "Comparing versions", + "get_version": "Loading snapshot", + "update_deterministic_fields": "Updating course", + "get_attachment_text": "Reading attachment", + "list_specializations": "Loading specializations", + "define_specialization": "Creating specialization", + "assign_elective_to_tracks": "Categorizing elective", + "get_course_assignments": "Reading elective assignments", + "categorize_elective": "Categorizing elective", + "fetch_url": "Fetching URL", + "web_search": "Searching the web", + "signal_done": "Finalizing", + "get_document_draft": "Reading document draft", + "create_curriculum_version": "Creating snapshot", + "get_course_draft": "Reading draft", + "remove_elective_from_tracks": "Removing elective", + "ask_user": "Asking user", + "get_preview_url": "Getting preview URL", + "list_courses": "Looking up courses", + "update_agent_draft": "Updating draft", +} + + +def tool_status_label(name: str) -> str: + return _TOOL_LABELS.get(name, name.replace("_", " ").title()) + + +class OpenRouterError(RuntimeError): + def __init__(self, status_code: int, retry_after: str | None = None, message: str | None = None, provider_message: str = ""): + self.status_code = status_code + self.retry_after = retry_after + self._message = message + self.provider_message = provider_message + super().__init__(self.message) + + @property + def message(self) -> str: + if self._message: + return self._message + if self.status_code in {429, 503}: + if self.retry_after: + return f"Model provider is rate limited. Try again in {self.retry_after} seconds." + return "Model provider is rate limited. Please try again shortly." + return "Model provider request failed. Please try again later." + + +def _messages(system: str, messages: list[dict]) -> list[dict]: + return [{"role": "system", "content": system}, *messages] + + +def _headers() -> dict: + return {"Authorization": f"Bearer {KEY}"} + + +def _message_content(data: dict) -> str: + try: + return _assistant_message(data)["content"].strip() + except (KeyError, TypeError, AttributeError) as exc: + raise OpenRouterError(502, message="Model provider returned an invalid response. Please try again later.", provider_message=str(data.get("error") or data)[:500]) from exc + + +def _raise_for_status(response) -> None: + try: + response.raise_for_status() + except HTTPStatusError as exc: + raise OpenRouterError( + exc.response.status_code, + exc.response.headers.get("retry-after"), + provider_message=exc.response.text[:500], + ) from exc + + +_RETRYABLE = (httpx.ReadError, httpx.ConnectError, httpx.RemoteProtocolError, httpx.TimeoutException) +_RETRYABLE_STATUS = {502, 503} +_MAX_RETRIES = 3 +_FIB = [1, 1, 2, 3, 5, 8, 13] + + +def _retry_sleep(attempt: int, retry_after: str | None = None) -> None: + if retry_after: + try: + time.sleep(min(float(retry_after), 30)) + return + except (ValueError, TypeError): + pass + base = _FIB[min(attempt, len(_FIB) - 1)] + jitter = base * 0.25 * (2 * random.random() - 1) + time.sleep(max(0.5, base + jitter)) + + +def _provider_error(data: dict) -> str: + error = data.get("error") if isinstance(data, dict) else None + if isinstance(error, dict): + return str(error.get("message") or error) + return str(error or data)[:500] + + +def _public_shape_error(provider_message: str) -> str: + lowered = provider_message.lower() + if "tool" in lowered and ("support" in lowered or "unsupported" in lowered): + return "The selected model does not support editor tools. Switch to a tool-calling model and try again." + if "rate" in lowered or "quota" in lowered: + return "Model provider is rate limited. Please try again shortly." + return "Model provider returned an invalid response. Please try again later." + + +def _assistant_message(data: dict) -> dict: + try: + message = data["choices"][0]["message"] + except (KeyError, IndexError, TypeError) as exc: + provider_message = _provider_error(data) + raise OpenRouterError(502, message=_public_shape_error(provider_message), provider_message=provider_message) from exc + if not isinstance(message, dict): + raise OpenRouterError(502, message="Model provider returned an invalid response. Please try again later.", provider_message=str(message)[:500]) + return message + + +def _chat_message(messages: list[dict], tools: list[dict], model: str = MODEL) -> tuple[dict, int]: + with Client(timeout=120) as client: + response = client.post(URL, headers=_headers(), json={"model": model, "messages": messages, "tools": tools}) + _raise_for_status(response) + data = response.json() + prompt_tokens = (data.get("usage") or {}).get("prompt_tokens") or 0 + return _assistant_message(data), prompt_tokens + + +def _chat_with_tools(messages: list[dict], tools: list[dict], tool_runner, on_tool_result=None): + last_prompt_tokens = 0 + active_model = MODEL + switched_to_fallback = False + + for i in range(50): + yield {"$status": f"Thinking (step {i + 1})..."} + message = None + prompt_tokens = 0 + + models_to_try = [active_model] + + for model in models_to_try: + for attempt in range(_MAX_RETRIES): + try: + message, prompt_tokens = _chat_message(messages, tools, model=model) + break + except OpenRouterError as exc: + if exc.status_code == 429: + yield {"$status": f"Rate limited by {model}. {exc.message}"} + logger.warning("Rate limited by %s (attempt %d/%d): %s", model, attempt + 1, _MAX_RETRIES, exc.message) + raise + if exc.status_code in _RETRYABLE_STATUS: + logger.warning("Provider %s error %d (attempt %d/%d): %s", model, exc.status_code, attempt + 1, _MAX_RETRIES, exc.message) + if attempt < _MAX_RETRIES - 1: + yield {"$status": f"Server error ({exc.status_code}). Retrying {model} (attempt {attempt + 2}/{_MAX_RETRIES})..."} + _retry_sleep(attempt, exc.retry_after) + continue + raise + except Exception as exc: + logger.warning("Network error with %s (attempt %d/%d): %s", model, attempt + 1, _MAX_RETRIES, exc) + if attempt < _MAX_RETRIES - 1: + yield {"$status": f"Connection error. Retrying {model} (attempt {attempt + 2}/{_MAX_RETRIES})..."} + _retry_sleep(attempt) + continue + + if message is not None: + break + + if model == MODEL and FALLBACK_MODEL and not switched_to_fallback: + switched_to_fallback = True + active_model = FALLBACK_MODEL + yield {"$status": "Primary model unavailable. Switching to backup model..."} + logger.info("Primary model %s failed after %d attempts, switching to %s", MODEL, _MAX_RETRIES, FALLBACK_MODEL) + models_to_try.append(FALLBACK_MODEL) + + if message is None: + raise OpenRouterError(502, message="All model providers failed after retries. Please try again later.") + + if prompt_tokens: + last_prompt_tokens = prompt_tokens + + tool_calls = message.get("tool_calls") or [] + if not tool_calls: + text = str(message.get("content") or "").strip() + if text: + yield text + elif i > 0: + yield "The agent finished without a response. Try rephrasing your request or breaking it into smaller steps." + if last_prompt_tokens: + yield {"$usage": {"prompt_tokens": last_prompt_tokens, "context_length": context_length()}} + return + messages.append({"role": "assistant", "content": message.get("content") or "", "tool_calls": tool_calls}) + + ack_text = str(message.get("content") or "").strip() + if ack_text: + yield ack_text + + done = False + result = {} + for tool_call in tool_calls: + name = (tool_call.get("function") or {}).get("name") or "" + try: + arguments = _tool_arguments(tool_call) + except (ValueError, json.JSONDecodeError) as exc: + result = {"error": f"Invalid tool arguments: {exc}"} + yield {"$status": f"{tool_status_label(name)}..."} + yield {"$event": "tool_result", "name": name, "status": "error"} + messages.append({"role": "tool", "tool_call_id": tool_call.get("id") or name, "name": name, "content": json.dumps(result, ensure_ascii=False)}) + continue + yield {"$status": f"{tool_status_label(name)}..."} + yield {"$event": "tool_call", "name": name, "arguments": arguments} + try: + result = _run_tool(tool_runner, name, arguments) + except ValueError as exc: + result = {"error": str(exc)} + yield {"$event": "tool_result", "name": name, "status": "ok" if "error" not in result else "error"} + if on_tool_result: + on_tool_result(name, result) + content = json.dumps(result, ensure_ascii=False) + if len(content) > 50000: + content = content[:50000] + '... [truncated; use semester filter or batch_read_courses for smaller results]' + messages.append( + { + "role": "tool", + "tool_call_id": tool_call.get("id") or name, + "name": name, + "content": content, + } + ) + if (result or {}).get("done"): + done = True + if done: + summary = str((result or {}).get("summary") or "").strip() + if summary: + yield summary + if last_prompt_tokens: + yield {"$usage": {"prompt_tokens": last_prompt_tokens, "context_length": context_length()}} + return + yield "I created tool results, but could not finish the response. Please check the Review panel." + if last_prompt_tokens: + yield {"$usage": {"prompt_tokens": last_prompt_tokens, "context_length": context_length()}} + + +def _tool_arguments(tool_call: dict) -> dict: + raw = (tool_call.get("function") or {}).get("arguments") or "{}" + arguments = json.loads(raw) if isinstance(raw, str) else raw + if not isinstance(arguments, dict): + raise ValueError("Tool arguments must be an object") + return arguments + + +def _run_tool(tool_runner, name: str, arguments: dict) -> dict: + try: + return tool_runner(name, arguments) + except Exception as exc: + return {"error": str(exc)} + + +def _log_stream_fallback(exc: Exception) -> None: + if isinstance(exc, HTTPStatusError): + response = exc.response + logger.warning( + "OpenRouter streaming failed; retrying without stream. status=%s model=%s body=%s", + response.status_code, + MODEL, + response.text[:500], + ) + return + logger.warning("OpenRouter streaming failed; retrying without stream. error=%s model=%s", exc.__class__.__name__, MODEL) + + +def _stream_chat(messages: list[dict], model: str = MODEL): + payload = {"model": model, "messages": messages, "stream": True} + last_exc = None + for attempt in range(_MAX_RETRIES): + try: + with Client(timeout=120) as client: + with client.stream("POST", URL, headers=_headers(), json=payload) as response: + if response.is_error: + response.read() + response.raise_for_status() + for line in response.iter_lines(): + token = _stream_token(line) + if token: + yield token + return + except _RETRYABLE as exc: + last_exc = exc + logger.warning("Network error in stream with %s (attempt %d/%d): %s", model, attempt + 1, _MAX_RETRIES, exc) + if attempt < _MAX_RETRIES - 1: + _retry_sleep(attempt) + except OpenRouterError as exc: + last_exc = exc + if exc.status_code in _RETRYABLE_STATUS: + logger.warning("Provider %s error %d in stream (attempt %d/%d): %s", model, exc.status_code, attempt + 1, _MAX_RETRIES, exc.message) + if attempt < _MAX_RETRIES - 1: + _retry_sleep(attempt, exc.retry_after) + continue + raise + if model == MODEL and FALLBACK_MODEL: + logger.info("Primary model %s failed in stream, falling back to %s", MODEL, FALLBACK_MODEL) + yield from _stream_chat(messages, model=FALLBACK_MODEL) + return + raise OpenRouterError(502, message="Network error connecting to model provider. Please try again.") from last_exc + + +def _chat_text(messages: list[dict], model: str = MODEL) -> str: + last_exc = None + for attempt in range(_MAX_RETRIES): + try: + with Client(timeout=120) as client: + response = client.post(URL, headers=_headers(), json={"model": model, "messages": messages}) + _raise_for_status(response) + return _message_content(response.json()) + except _RETRYABLE as exc: + last_exc = exc + logger.warning("Network error in chat_text with %s (attempt %d/%d): %s", model, attempt + 1, _MAX_RETRIES, exc) + if attempt < _MAX_RETRIES - 1: + _retry_sleep(attempt) + except OpenRouterError as exc: + last_exc = exc + if exc.status_code in _RETRYABLE_STATUS: + logger.warning("Provider %s error %d in chat_text (attempt %d/%d): %s", model, exc.status_code, attempt + 1, _MAX_RETRIES, exc.message) + if attempt < _MAX_RETRIES - 1: + _retry_sleep(attempt, exc.retry_after) + continue + raise + if model == MODEL and FALLBACK_MODEL: + logger.info("Primary model %s failed in chat_text, falling back to %s", MODEL, FALLBACK_MODEL) + return _chat_text(messages, model=FALLBACK_MODEL) + raise OpenRouterError(502, message="Network error connecting to model provider. Please try again.") from last_exc + + +def call(system: str, user: str, model: str = MODEL) -> dict: + last_exc = None + for attempt in range(_MAX_RETRIES): + try: + with Client(timeout=120) as c: + r = c.post( + URL, + headers=_headers(), + json={"model": model, "messages": _messages(system, [{"role": "user", "content": user}])}, + ) + _raise_for_status(r) + data = r.json() + text = _message_content(data) + text = text.strip().removeprefix("```json").removeprefix("```").removesuffix("```").strip() + return json.loads(text) + except _RETRYABLE as exc: + last_exc = exc + logger.warning("Network error in call with %s (attempt %d/%d): %s", model, attempt + 1, _MAX_RETRIES, exc) + if attempt < _MAX_RETRIES - 1: + _retry_sleep(attempt) + except OpenRouterError as exc: + last_exc = exc + if exc.status_code in _RETRYABLE_STATUS: + logger.warning("Provider %s error %d in call (attempt %d/%d): %s", model, exc.status_code, attempt + 1, _MAX_RETRIES, exc.message) + if attempt < _MAX_RETRIES - 1: + _retry_sleep(attempt, exc.retry_after) + continue + raise + if model == MODEL and FALLBACK_MODEL: + logger.info("Primary model %s failed in call, falling back to %s", MODEL, FALLBACK_MODEL) + return call(system, user, model=FALLBACK_MODEL) + raise OpenRouterError(502, message="Network error connecting to model provider. Please try again.") from last_exc + + +def stream_chat(system: str, messages: list[dict], tools: list[dict] | None = None, tool_runner=None, on_tool_result=None): + chat_messages = _messages(system, messages) + if tools and tool_runner: + yield from _chat_with_tools(chat_messages, tools, tool_runner, on_tool_result) + return + + emitted = False + try: + for token in _stream_chat(chat_messages): + emitted = True + yield token + except (HTTPError, json.JSONDecodeError) as exc: + _log_stream_fallback(exc) + + if emitted: + return + + text = _chat_text(chat_messages) + if text: + yield text + + +def _stream_token(line: str) -> str: + if not line or line.startswith(":"): + return "" + if not line.startswith("data:"): + return "" + data = line.removeprefix("data:").strip() + if data == "[DONE]": + return "" + chunk = json.loads(data) + choice = (chunk.get("choices") or [{}])[0] + delta = choice.get("delta") or {} + return delta.get("content") or "" + + +try: + fetch_context_length() +except Exception: + pass diff --git a/backend/app/services/refinement.py b/backend/app/services/refinement.py new file mode 100644 index 0000000000000000000000000000000000000000..ce6fc2b5f2e1d43ab99844ea689e6fb1a9f85aca --- /dev/null +++ b/backend/app/services/refinement.py @@ -0,0 +1,630 @@ +import logging +import re + +from app.supabase import first_row, supabase +from app.services.books import parse_books, raw_book_section +from app.services.curriculum import invalidate_curriculum_cache +from app.services.deterministic import compute_hours, compute_program, compute_course_type +from app.services.openrouter import call as llm + +logger = logging.getLogger(__name__) + +WORD = re.compile(r"\w+") +UNIT_LINE = re.compile(r"\b(Unit\s+\d+|Module\s+(?:\d+|[IVX]+))\s*[:.-]?\s*(.*)$", re.IGNORECASE) +HOURS_SUFFIX = re.compile(r"\s*\b\d+\s+Hours?\b\s*$", re.IGNORECASE) +DESIRABLE_LINE = re.compile(r"\bDesirable\s+([^\n]+)", re.IGNORECASE) +COURSE_CODE = re.compile(r"\bCourse\s+Code\s*:?\s*([A-Z0-9]+)", re.IGNORECASE) +PAGE_NOISE = re.compile( + r"P\.?\s*E\.?\s*S\.?\s*University|" + r"Curriculum|" + r"\s*:-\s*[A-Za-z]*\s*\d{4}\s*-\s*\d{4}\b|" + r"\b\d+\s*\|\s*Page\b", + re.IGNORECASE, +) + +SYS = """You refine PES University course submissions for the UG curriculum template. +Return only valid JSON. No markdown. No commentary. + +Your job: take a raw faculty submission and produce a clean, template-ready course record. + +CRITICAL -- you MUST fix all of the following in every field: +- Spelling errors (e.g. "lerning" -> "learning", "neual" -> "neural", "introducton" -> "introduction") +- Title case for the course title (e.g. "machine lerning" -> "Machine Learning") +- Title case for unit titles (e.g. "Unit 1: Introducton to ML" -> "Unit 1: Introduction to ML") +- Grammar and punctuation in all text fields +- Capitalize the first letter of every sentence in content fields +- Fix malformed abbreviations (e.g. "ML" is fine, but "ml" should be "ML") + +Preserve the academic content, topics, and structure. Do not invent new topics, remove submitted topics, or change the course scope. But every string you output must be free of spelling errors and properly formatted. + +Editing rules: +- If prelude, objectives, outcomes, units, tools, or books are already present, clean them instead of replacing them. +- Generate a field only when that field is missing, incomplete, or too rough for the template. +- Preserve every syllabus topic and subtopic from Raw Course Content. +- Do not summarize away, omit, replace, merge away, or simplify syllabus topics. +- Do not add advanced, fashionable, or unrelated topics. +- Do not invent course codes, books, references, departments, credits, or external prerequisites. +- Keep objectives and outcomes direct, concise, and aligned with the submitted scope. +- Return exactly 4 units. If the input already has 4 units, keep those boundaries. If it has a different structure, redistribute topics in order without dropping content. +- Unit hours must sum to the supplied total unit hours. +- For tools/languages, use Preferred Tools / Languages when provided; otherwise identify course-specific tools, languages, platforms, or AI tools from Raw Course Content. +- Do not use canned defaults for tools/languages. +- For desirable knowledge, use only relevant knowledge from Previously Completed Courses. Return an empty string when none apply. +- Generate laboratory experiments only when practical hours are non-zero. Keep them tied to submitted topics. +- Copy and clean books from the submitted book fields only. +- Use empty strings or empty arrays for truly unavailable optional content. Do not output "-". +""" + +SCHEMA = """{ + "course_title": "corrected course title", + "prelude": "one short paragraph", + "objectives": ["3 to 4 objectives"], + "course_outcomes": ["3 to 4 measurable outcomes"], + "units": [{"title": "Unit 1: Title", "content": "compact topic list", "hours": 14}], + "lab_experiments": ["concise lab item"], + "tools_languages": "course-specific tools, languages, platforms, or AI tools", + "desirable_knowledge": "short text based only on previously completed courses, or empty string", + "text_books": ["submitted text books only"], + "reference_books": ["submitted reference books only"] +}""" + +EXAMPLES = """Behavior examples calibrated to PESU curriculum style. +Use them only to learn fidelity, field shape, and level of detail. Do not copy example facts into the real answer. + +Example 1: already refined input +Input: +Course Title: Web Technologies +Weekly Hours: L 4, T 0, P 0, S 4, C 4 +Total Unit Hours: 56 +Raw Course Content: +Unit 1: HTML, CSS and Client-Side Scripting - web architecture, HTTP request and response formats, URLs, HTML elements and attributes, web forms, HTML5 tags and controls, CSS selectors, style properties, box model, JavaScript objects, DOM manipulation, events and event handling. 14 Hours +Unit 2: HTML5 and ReactJS - HTML5 APIs, audio, video, progress, geolocation, callbacks, promises, single page applications, XML vs JSON, async/await, JSX, rendering elements, React setup, components, styling, props, state and context. 14 Hours +Unit 3: ReactJS and NodeJS - complex state management, keys, event handling, forms, hooks including useState, useRef, useEffect, useContext and useReducer, React Router, introduction to NextJS, NodeJS architecture, callbacks, modules, buffers, streams, file system and Axios API. 14 Hours +Unit 4: MongoDB and ExpressJS - documents, collections, reading and writing to MongoDB, MongoDB NodeJS driver, running a React application on NodeJS, React Router, web services, REST APIs, Express routing, URL building, error handling, middleware, form data and file upload. 14 Hours +Preferred Tools / Languages: HTML, CSS, JavaScript, MERN Technologies, GitHub, AI tools: Copilot and Tabnine +Previously Completed Courses: None +Text Books: +1. Robin Nixon, Learning PHP, MySQL and JavaScript, 5th Edition, O'Reilly Media, 2018. +2. Vasan Subramanian, Pro MERN Stack: Full Stack Web App Development with Mongo, Express, React, and Node, Apress, 2017. +Output: +{ + "course_title": "Web Technologies", + "prelude": "This course develops an in-depth understanding of technologies required to design and develop rich web applications using client-side scripting, ReactJS, NodeJS, ExpressJS, and MongoDB.", + "objectives": ["Build web pages using HTML, CSS, JavaScript, and advanced JavaScript concepts.", "Use HTML5, AJAX, JSON, and ReactJS to develop interactive user interfaces.", "Build multi-tier applications by connecting ReactJS interfaces to NodeJS services.", "Integrate MongoDB through ExpressJS and RESTful web services."], + "course_outcomes": ["Develop web pages using HTML, CSS, JavaScript, DOM manipulation, and event handling.", "Create interactive user interfaces using HTML5 APIs and ReactJS components.", "Implement server-side functionality using NodeJS, modules, streams, file system APIs, and routing.", "Create RESTful services with ExpressJS and integrate MongoDB for persistent storage."], + "units": [ + {"title": "Unit 1: HTML, CSS and Client-Side Scripting", "content": "Web architecture, HTTP request and response formats, URLs, HTML elements and attributes, web forms, HTML5 tags and controls, CSS selectors, style properties, box model, JavaScript objects, DOM manipulation, events and event handling.", "hours": 14}, + {"title": "Unit 2: HTML5 and ReactJS", "content": "HTML5 APIs, audio, video, progress, geolocation, callbacks, promises, single page applications, XML vs JSON, async/await, JSX, rendering elements, React setup, components, styling, props, state and context.", "hours": 14}, + {"title": "Unit 3: ReactJS and NodeJS", "content": "Complex state management, keys, event handling, forms, hooks including useState, useRef, useEffect, useContext and useReducer, React Router, introduction to NextJS, NodeJS architecture, callbacks, modules, buffers, streams, file system and Axios API.", "hours": 14}, + {"title": "Unit 4: MongoDB and ExpressJS", "content": "Documents, collections, reading and writing to MongoDB, MongoDB NodeJS driver, running a React application on NodeJS, React Router, web services, REST APIs, Express routing, URL building, error handling, middleware, form data and file upload.", "hours": 14} + ], + "lab_experiments": [], + "tools_languages": "HTML, CSS, JavaScript, MERN Technologies, GitHub; AI tools: Copilot, Tabnine", + "desirable_knowledge": "", + "text_books": ["Robin Nixon, Learning PHP, MySQL and JavaScript, 5th Edition, O'Reilly Media, 2018.", "Vasan Subramanian, Pro MERN Stack: Full Stack Web App Development with Mongo, Express, React, and Node, Apress, 2017."], + "reference_books": [] +} + +Example 2: detailed lab-integrated input with title and formatting issues +Input: +Course Title: data structures & its aplications +Weekly Hours: L 4, T 0, P 2, S 5, C 5 +Total Unit Hours: 56 +Raw Course Content: +Unit 1: Linked List and Stacks - Review of C, static and dynamic memory allocation, doubly linked list, circular linked list, multilist and sparse matrix, skip list dictionary case study, stack using arrays and linked list, function execution, nested functions, recursion, Tower of Hanoi, infix to postfix, infix to prefix, expression evaluation, matching parenthesis. 14 Hours +Unit 2: Queues and Trees - simple queue, circular queue, priority queue, dequeue using arrays and linked list, Josephus problem, CPU scheduling using queues, N-ary trees, binary trees, binary search trees, forests, conversion to binary tree, preorder, inorder and postorder traversal. 14 Hours +Unit 3: Application of Trees and Introduction to Graphs - BST insertion and deletion using arrays and dynamic allocation, binary expression tree, threaded binary search tree, heaps, priority queue using min heap and max heap, dictionary and decision tree applications, AVL trees, rotations, splay tree, graph properties, adjacency matrix, adjacency list, DFS, BFS, network topology representation. 14 Hours +Unit 4: Applications of Graphs, B-Trees, Suffix Tree and Hashing - BFS and DFS applications, connectivity, path finding in a network, suffix trees, trie trees, insert, delete and search operations, hashing, hash functions, hash tables, separate chaining, open addressing, double hashing, rehashing, URL decoding and word prediction using trie trees and suffix trees. 14 Hours +Laboratory: linked list operations; stack applications; queue applications; binary tree and BST applications; graph data structure applications; hashing techniques. +Preferred Tools / Languages: C-Programming language; AI tools: VisuAlgo (Interactive Visualizations), Algorithm Visualizer (AI Explanations) +Previously Completed Courses: Problem Solving with C +Text Books: 1. Langsam Yedidyah, Moshe J. Augenstein, Aaron M. Tenenbaum, Data Structures using C / C++, Pearson Education Inc., 2nd Edition, 2015. +Output: +{ + "course_title": "Data Structures and Applications", + "prelude": "This course introduces fundamental data structure concepts with emphasis on their theoretical foundations, implementation techniques, and practical applications using a programming language.", + "objectives": ["Analyze and design data structures for efficient storage, retrieval, and manipulation.", "Use linked lists, stacks, queues, trees, heaps, and graphs for suitable computational tasks.", "Implement insertion, deletion, searching, and modification operations across linear and non-linear data structures.", "Select and apply appropriate data structures to solve application-oriented problems."], + "course_outcomes": ["Select and apply appropriate data structures for solving problems across application domains.", "Implement fundamental data structures and their operations using suitable programming constructs.", "Use data structures effectively to design efficient solutions for computational problems.", "Develop software components by applying data structure principles and their applications."], + "units": [ + {"title": "Unit 1: Linked List and Stacks", "content": "Review of C, static and dynamic memory allocation, doubly linked list, circular linked list, multilist and sparse matrix, skip list dictionary case study, stack using arrays and linked list, function execution, nested functions, recursion, Tower of Hanoi, infix to postfix, infix to prefix, expression evaluation, matching parenthesis.", "hours": 14}, + {"title": "Unit 2: Queues and Trees", "content": "Simple queue, circular queue, priority queue, dequeue using arrays and linked list, Josephus problem, CPU scheduling using queues, N-ary trees, binary trees, binary search trees, forests, conversion to binary tree, preorder, inorder and postorder traversal.", "hours": 14}, + {"title": "Unit 3: Application of Trees and Introduction to Graphs", "content": "BST insertion and deletion using arrays and dynamic allocation, binary expression tree, threaded binary search tree, heaps, priority queue using min heap and max heap, dictionary and decision tree applications, AVL trees, rotations, splay tree, graph properties, adjacency matrix, adjacency list, DFS, BFS, network topology representation.", "hours": 14}, + {"title": "Unit 4: Applications of Graphs, B-Trees, Suffix Tree and Hashing", "content": "BFS and DFS applications, connectivity, path finding in a network, suffix trees, trie trees, insert, delete and search operations, hashing, hash functions, hash tables, separate chaining, open addressing, double hashing, rehashing, URL decoding and word prediction using trie trees and suffix trees.", "hours": 14} + ], + "lab_experiments": ["Linked list and advanced operations.", "Stack applications.", "Queue applications.", "Binary tree and binary search tree applications.", "Graph data structure applications.", "Hashing techniques."], + "tools_languages": "C-Programming language; AI tools: VisuAlgo (Interactive Visualizations), Algorithm Visualizer (AI Explanations)", + "desirable_knowledge": "Problem Solving with C", + "text_books": ["Langsam Yedidyah, Moshe J. Augenstein, Aaron M. Tenenbaum, Data Structures using C / C++, Pearson Education Inc., 2nd Edition, 2015."], + "reference_books": [] +} +""" + + +def _lines(value) -> list[str]: + if not value: + return [] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + return [line.strip() for line in str(value).splitlines() if line.strip()] + + +def _clean_noise(value: str) -> str: + return re.sub(r"\s+", " ", PAGE_NOISE.sub(" ", value)).strip() + + +def _books(*values) -> list[str]: + return parse_books(*values) + + +def _raw_book_section(raw_content: str, kind: str) -> str: + return raw_book_section(raw_content, kind) + + +def _words(text: str) -> int: + return len(WORD.findall(text)) + + +def _clean_part(value: str) -> str: + return _clean_noise(value.strip(" \t-*\u2022")) + + +def _split_parts(text: str) -> list[str]: + lines = [_clean_part(line) for line in text.splitlines() if _clean_part(line)] + if len(lines) >= 4: + return lines + compact = _clean_part(text) + parts = [_clean_part(part) for part in re.split(r"(?<=[.!?])\s+", compact) if _clean_part(part)] + if len(parts) >= 4: + return parts + words = compact.split() + if not words: + return [] + return [ + " ".join(words[(index * len(words)) // 4 : ((index + 1) * len(words)) // 4]) + for index in range(4) + ] + + +def _four_units_from_raw(raw_content: str) -> list[dict]: + parts = _split_parts(raw_content) + if not parts: + return [] + buckets = [[] for _ in range(4)] + for index, part in enumerate(parts): + buckets[min(index * 4 // len(parts), 3)].append(part) + return [ + {"title": f"Unit {index + 1}", "content": " ".join(bucket).strip(), "hours": 0} + for index, bucket in enumerate(buckets) + if bucket + ] + + +def _course_contents(raw_content: str) -> str: + lines = raw_content.splitlines() + start = None + for index, line in enumerate(lines): + if "Course" in line and "Contents" in line: + start = index + break + if "Course" in line and any("Contents" in item for item in lines[index + 1 : index + 3]): + start = index + break + if start is None: + return raw_content + + end = len(lines) + for index in range(start + 1, len(lines)): + marker = lines[index].strip() + if not marker: + continue + if any(marker.startswith(label) for label in ("Laboratory", "Text Book", "Reference", "Course Outcome", "Assignment /")): + end = index + break + return "\n".join(lines[start:end]).strip() + + +def _syllabus_line(line: str) -> str: + clean = _clean_part(line) + clean = re.sub(r"^(Course\s+Contents|Contents)\s+", "", clean, flags=re.IGNORECASE) + clean = re.sub(r"^Course\s+(?=Unit\s+\d+|Module\s+(?:\d+|[IVX]+))", "", clean, flags=re.IGNORECASE) + return HOURS_SUFFIX.sub("", clean).strip() + + +def _units_from_course_contents(raw_content: str, fallback: bool = True) -> list[dict]: + content = _course_contents(raw_content) + lines = [line.strip() for line in content.splitlines() if line.strip()] + units = [] + current = None + + for line in lines: + clean = _syllabus_line(line) + if not clean: + continue + match = UNIT_LINE.search(clean) + if match: + if current: + units.append(current) + label = match.group(1).strip().title() + rest = match.group(2).strip(" :-") + if " - " in rest: + title, inline_content = rest.split(" - ", 1) + elif _words(rest) > 12: + title, inline_content = "", rest + else: + title, inline_content = rest, "" + current = { + "title": f"{label}: {title.strip()}" if title.strip() else label, + "content": inline_content.strip(), + "hours": 0, + } + continue + if current: + current["content"] = f"{current['content']} {clean}".strip() + + if current: + units.append(current) + if len(units) == 4: + return units + return _four_units_from_raw(content) if fallback else [] + + +def _unit_text(unit: dict) -> str: + title = str(unit.get("title", "")).strip() + content = str(unit.get("content", "")).strip() + if title and content: + return f"{title}: {content}" + return title or content + + +def _fit_four_units(units: list[dict], raw_content: str) -> list[dict]: + raw = _course_contents(raw_content).strip() + raw_units = _units_from_course_contents(raw_content, fallback=False) + raw_unit_words = _words(" ".join(_unit_text(unit) for unit in raw_units)) + if raw_units and raw_unit_words >= _words(raw) * 0.7: + return raw_units + if raw and _words(" ".join(_unit_text(unit) for unit in units)) < _words(raw) * 0.8: + return raw_units or _units_from_course_contents(raw_content) + if len(units) > 4: + fourth = units[3].copy() + fourth["content"] = " ".join(_unit_text(unit) for unit in units[3:]).strip() + return units[:3] + [fourth] + if len(units) == 4: + return units + if raw: + return raw_units or _units_from_course_contents(raw_content) + return units + + +def _units(value) -> list[dict]: + units = [] + for item in value if isinstance(value, list) else []: + if not isinstance(item, dict): + continue + title = str(item.get("title", "")).strip() + content = str(item.get("content", "")).strip() + hours_raw = str(item.get("hours", "0")) + hours = int("".join(ch for ch in hours_raw if ch.isdigit()) or 0) + if title or content: + units.append({"title": title, "content": content, "hours": hours}) + return units + + +def _assign_hours(units: list[dict], total_hours: int) -> list[dict]: + if not units: + return units + for unit in units: + unit["hours"] = 14 + return units + + +def _text(*values) -> str: + for value in values: + if isinstance(value, str): + text = value.strip() + if text and text != "-": + return text + return "" + + +def _course_code(raw_content: str) -> str: + match = COURSE_CODE.search(raw_content or "") + return match.group(1).upper() if match else "" + + +def _prior_course_titles(sub: dict) -> list[str]: + semester = int(sub["semester"]) + if semester <= 1: + return [] + rows = ( + supabase.table("refined_submissions") + .select("submission_id,course_code,course_title,semester") + .lt("semester", semester) + .order("semester") + .execute() + .data + ) + ids = [row["submission_id"] for row in rows if row.get("submission_id")] + submissions = supabase.table("submissions").select("id,target_department,raw_course_content").in_("id", ids).execute().data if ids else [] + departments = {row["id"]: row.get("target_department") for row in submissions} + raw_codes = {row["id"]: _course_code(row.get("raw_course_content") or "") for row in submissions} + titles = [] + for row in rows: + if departments.get(row.get("submission_id")) != sub.get("target_department"): + continue + title = str(row.get("course_title") or "").strip() + code = str(row.get("course_code") or raw_codes.get(row.get("submission_id")) or "").strip() + label = f"{code} - {title}" if code else title + if title and label not in titles: + titles.append(label) + return titles + + +def _normalized(text: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", text.lower()).strip() + + +def _singularized(text: str) -> str: + return " ".join(word[:-1] if len(word) > 3 and word.endswith("s") else word for word in _normalized(text).split()) + + +def _core_key(text: str) -> str: + skip = {"a", "an", "and", "for", "in", "its", "of", "the", "to", "with"} + return " ".join(word for word in _singularized(text).split() if word not in skip) + + +def _prior_keys(course: str) -> list[str]: + match = re.match(r"\s*([A-Z]{2}\d{2}[A-Z0-9]+)\s*[-:]\s*(.+)", course) + code = match.group(1).lower() if match else "" + title = match.group(2) if match else course + return [key for key in (code, _normalized(title), _singularized(title), _core_key(title)) if key] + + +SPELLING_FIXES = { + "Introducton": "Introduction", + "introducton": "Introduction", + "Introdcution": "Introduction", + "Neual": "Neural", + "neual": "neural", + "Adavnced": "Advanced", + "adavnced": "advanced", + "Applicatons": "Applications", + "applicatons": "applications", + "preprocesing": "preprocessing", + "recurent": "recurrent", + "anomal": "anomaly", + "stuides": "studies", + "lerning": "learning", + "inteligence": "intelligence", + "artifical": "artificial", + "concurrncy": "concurrency", + "procesing": "processing", + "optimisation": "optimization", + "maximisation": "maximization", + "minimisation": "minimization", + "behaviour": "behavior", + "colour": "color", + " favour ": " favor ", + "analyse": "analyze", + "recognise": "recognize", + "summarise": "summarize", + "characterise": "characterize", + "utilise": "use", + "initialise": "initialize", + "licence": "license", + "defence": "defense", + "programme": "program", + "centre": "center", + "metre": "meter", + "litre": "liter", + "catalogue": "catalog", + "dialogue": "dialog", + "analysing": "analyzing", + "recognising": "recognizing", + "summarising": "summarizing", + "utilising": "using", + "initialising": "initializing", + "labelling": "labeling", + "modelling": "modeling", + "travelling": "traveling", + "cancelling": "canceling", + "fulfil ": "fulfill ", + "enrol ": "enroll ", + "skillset": "skill set", + "dataset": "data set", + "email ": "e-mail ", + "e mail": "e-mail", +} + + +def _fix_spelling(text: str) -> str: + if not text: + return text + for wrong, right in SPELLING_FIXES.items(): + text = text.replace(wrong, right) + return text + + +def _title_case(text: str) -> str: + if not text: + return text + words = text.split() + minor = {"a", "an", "the", "and", "but", "or", "for", "nor", "in", "on", "at", "to", "by", "of", "with", "vs"} + result = [] + for i, w in enumerate(words): + if i == 0 or w.lower() not in minor: + result.append(w[:1].upper() + w[1:] if w.islower() else w) + else: + result.append(w.lower()) + return " ".join(result) + + +def _capitalize_sentences(text: str) -> str: + if not text: + return text + sentences = re.split(r"(?<=[.!?])\s+", text) + return " ".join(s[0].upper() + s[1:] if s else s for s in sentences) + + +def _post_process(out: dict) -> dict: + if not out: + return out + + title = out.get("course_title") or "" + title = _fix_spelling(title) + title = _title_case(title) + out["course_title"] = title + + for field in ("prelude", "tools_languages", "desirable_knowledge"): + val = out.get(field) or "" + if val: + out[field] = _capitalize_sentences(_fix_spelling(val)) + + for field in ("objectives", "course_outcomes", "lab_experiments"): + items = out.get(field) or [] + if isinstance(items, list): + out[field] = [_capitalize_sentences(_fix_spelling(item)) for item in items] + + for unit in out.get("units") or []: + if isinstance(unit, dict): + unit["title"] = _fix_spelling(unit.get("title") or "") + unit["content"] = _capitalize_sentences(_fix_spelling(unit.get("content") or "")) + + for field in ("text_books", "reference_books"): + items = out.get(field) or [] + if isinstance(items, list): + out[field] = [_fix_spelling(item) for item in items] + + return out + + +def _prior_matches(text: str, prior_courses: list[str]) -> str: + normalized = _normalized(text) + singularized = _singularized(text) + core = _core_key(text) + matches = [course for course in prior_courses if any(key in normalized or key in singularized or key in core for key in _prior_keys(course))] + return ", ".join(matches) + + +def _submitted_desirable(raw_content: str, prior_courses: list[str]) -> str | None: + match = DESIRABLE_LINE.search(raw_content or "") + if not match: + return None + value = _text(match.group(1)) + if not value: + return "" + return _prior_matches(value, prior_courses) or None + + +def _desirable(value, prior_courses: list[str], raw_content: str = "") -> str: + if not prior_courses: + return "" + submitted = _submitted_desirable(raw_content, prior_courses) + if submitted is not None: + return submitted + return _prior_matches(_text(value), prior_courses) + + +def _courses_text(courses: list[str]) -> str: + return "\n".join(f"- {course}" for course in courses) if courses else "None" + + +def build_refined_payload(sub: dict, out: dict, prior_courses: list[str] | None = None) -> dict: + out = out or {} + prior_courses = prior_courses or [] + det = compute_hours(sub["credit_category"]) + total_unit_hours = det["lecture_hours"] * 14 + raw_content = sub.get("raw_course_content") or "" + text_book_source = _raw_book_section(raw_content, "text") or sub["text_books"] + reference_book_source = _raw_book_section(raw_content, "reference") or sub.get("reference_books") + units = _assign_hours(_fit_four_units(_units(out.get("units")), sub.get("raw_course_content") or ""), total_unit_hours) + + objectives = _lines(out.get("objectives"))[:4] + course_outcomes = _lines(out.get("course_outcomes"))[:4] or objectives + + from app.services.elective_categorization import is_elective_course + + code = _course_code(raw_content) or sub.get("course_code", "") + is_elective = is_elective_course({"course_code": code, "semester": sub["semester"]}) + return { + "submission_id": sub["id"], + "semester": int(sub["semester"]), + "course_code": code, + "course_title": sub["course_title"], + "program": compute_program(sub["target_department"]), + "course_type": "Elective" if is_elective else compute_course_type(sub["credit_category"]), + "is_elective": is_elective, + **det, + "prelude": _text(out.get("prelude"), f"This course covers {sub['course_title'].strip()}."), + "objectives": objectives, + "course_outcomes": course_outcomes, + "units": units, + "lab_experiments": _lines(out.get("lab_experiments"))[:10] if det["practical_hours"] else [], + "tools_languages": _text(out.get("tools_languages"), sub.get("preferred_tools")), + "desirable_knowledge": _desirable(out.get("desirable_knowledge"), prior_courses, raw_content), + "text_books": _books(text_book_source), + "reference_books": _books(reference_book_source), + "status": "refined", + } + + +def refine(submission_id: int): + sub = first_row(supabase.table("submissions").select("*").eq("id", submission_id)) + if not sub: + raise LookupError("Submission not found") + det = compute_hours(sub["credit_category"]) + ctype = compute_course_type(sub["credit_category"]) + total_unit_hours = det["lecture_hours"] * 14 + prior_courses = _prior_course_titles(sub) + + prompt = f"""Return JSON matching this schema. Include every key: +{SCHEMA} + +{EXAMPLES} + +Now refine the real submission below. + +Course Title: {sub["course_title"]} +Offering Department: {sub["offering_department"]} +Target Department: {sub["target_department"]} +Semester: {sub["semester"]} +Credit Category: {sub["credit_category"]} +Course Type: {ctype} +Weekly Hours: L {det["lecture_hours"]}, T {det["tutorial_hours"]}, P {det["practical_hours"]}, S {det["self_study"]}, C {det["credits"]} +Total Unit Hours: {total_unit_hours} +Raw Course Content: +{sub["raw_course_content"]} + +Previously Completed Courses: +{_courses_text(prior_courses)} + +Text Books: +{sub["text_books"]} + +Reference Books: +{sub.get("reference_books") or "-"} + +Preferred Tools / Languages: +{sub.get("preferred_tools") or "-"}""" + + out = llm(SYS, prompt) + merged = build_refined_payload(sub, out, prior_courses) + _post_process(merged) + + existing = supabase.table("refined_submissions").select("id").eq("submission_id", submission_id).execute().data + if existing: + supabase.table("refined_submissions").update(merged).eq("submission_id", submission_id).execute() + refined_id = existing[0]["id"] + else: + refined_id = supabase.table("refined_submissions").insert(merged).execute().data[0]["id"] + + supabase.table("submissions").update({"status": "refined"}).eq("id", submission_id).execute() + invalidate_curriculum_cache() + + if merged["is_elective"]: + try: + from app.services.elective_categorization import categorize_refined_elective + categorize_refined_elective(int(refined_id)) + except Exception: + logger.exception("Elective categorization failed for refined_id=%s", refined_id) + + return merged diff --git a/backend/app/services/schema.py b/backend/app/services/schema.py new file mode 100644 index 0000000000000000000000000000000000000000..663f22b6011602eda10ac45405f3308e1c651fa8 --- /dev/null +++ b/backend/app/services/schema.py @@ -0,0 +1,30 @@ +from postgrest.exceptions import APIError + +from app.supabase import supabase + +REQUIRED_TABLES = ( + "submissions", + "refined_submissions", + "curriculum_versions", + "finalized_submissions", + "agent_document_drafts", + "agent_drafts", + "course_revision_history", + "chat_sessions", + "chat_messages", + "chat_attachments", + "specialization_definitions", + "course_specialization_assignments", +) + + +def schema_status() -> dict: + missing = [] + for table in REQUIRED_TABLES: + try: + supabase.table(table).select("id").limit(1).execute() + except APIError as exc: + if "schema cache" not in str(exc): + raise + missing.append(table) + return {"ok": not missing, "missing_tables": missing, "required_tables": list(REQUIRED_TABLES)} diff --git a/backend/app/supabase.py b/backend/app/supabase.py new file mode 100644 index 0000000000000000000000000000000000000000..262f9c6338fd1f578208f85613651e58a013cd87 --- /dev/null +++ b/backend/app/supabase.py @@ -0,0 +1,24 @@ +from os import environ +from supabase import create_client, ClientOptions +from dotenv import load_dotenv +import httpx + +load_dotenv("../.env") + +url = environ["SUPABASE_URL"].strip() +key = environ["SUPABASE_KEY"].strip() +supabase = create_client( + supabase_url=url, + supabase_key=key, + options=ClientOptions( + httpx_client=httpx.Client( + http2=False, + timeout=httpx.Timeout(connect=10, read=30, write=10, pool=10), + ), + ), +) + + +def first_row(query) -> dict | None: + rows = query.limit(1).execute().data + return rows[0] if rows else None diff --git a/backend/app/templates/jinja_1_to_8.html b/backend/app/templates/jinja_1_to_8.html new file mode 100644 index 0000000000000000000000000000000000000000..070a5961f82dfc71676b045a1a9bfbb2939764aa --- /dev/null +++ b/backend/app/templates/jinja_1_to_8.html @@ -0,0 +1,236 @@ +{% macro course_row(course, number) -%} +| Sl. No. | +Course Code | +Course Title | +Hours per week | +Credits | +AI Tools / Tools / Languages |
+ Course Type | +|||
|---|---|---|---|---|---|---|---|---|---|
| L | +T | +P | +S | +C | +|||||
| Total | +{{ total_cell(totals.lecture, totals.lecture_credit) }} | +{{ totals.tutorial }} | +{{ totals.practical }} | +{{ totals.self_study }} | +{{ totals.credits }} | ++ | + | ||
| Sl. No. | +Course Code | +Course Title | +Hours/week | +Credits | +Course Type | +|||
|---|---|---|---|---|---|---|---|---|
| L | +T | +P | +S | +C | +||||
| Total | +{{ totals.lecture }} | +{{ totals.tutorial }} | +{{ totals.practical }} | +{{ totals.self_study }} | +{{ totals.credits }} | ++ | ||
{{ unit_entry.unit.content }}
+{{ unit_entry.unit.content }}
+{{ unit_entry.unit.content }}
+{{ unit_entry.unit.content }}
+{{ unit_entry.unit.hours }} Hours
+{{ unit.title }}
+{{ unit.content }}
+{{ unit.hours }} Hours
+| Course Code | + {{ diff_cell(course_diff.course_code, proposed_course.course_code | course_code_for_year(proposed_course.semester, curriculum_year)) }} +Course Title | +{{ diff_inline(course_diff.course_title, proposed_course.course_title) }} | +|||||
|---|---|---|---|---|---|---|---|
| Program | +{{ proposed_course.program }} | +Hours per week / Credit Assigned | +L | T | P | S | C | +
| {{ proposed_course.lecture_hours }} | +{{ proposed_course.tutorial_hours }} | +{{ proposed_course.practical_hours }} | +{{ proposed_course.self_study }} | +{{ proposed_course.credits }} | +|||
| Semester | +{{ proposed_course.semester }} | +Type of Course | +{{ proposed_course.course_type }} | +||||
| AI Tools / Tools / Languages | + {{ diff_cell(course_diff.tools_languages, proposed_course.tools_languages) }} +Desirable Knowledge | +{{ diff_inline(course_diff.desirable_knowledge, proposed_course.desirable_knowledge) }} | +|||||
| Prelude | +{{ diff_inline(course_diff.prelude, proposed_course.prelude) }} | +
|---|---|
| Course Objectives: | +{{ render_list(course_diff, proposed_course, "objectives") }} | +
| Course Contents | +{{ render_units(course_diff, proposed_course) }} | +
| Laboratory | +{{ render_list(course_diff, proposed_course, "lab_experiments", ordered=true) }} | +
| Text Book(s): | +{{ render_list(course_diff, proposed_course, "text_books", ordered=true) }} | +
| Reference Book(s): | +{{ render_list(course_diff, proposed_course, "reference_books", ordered=true) }} | +
| Course Outcome | +{{ render_list(course_diff, proposed_course, "course_outcomes") }} | +
P.E.S. University
+Curriculum :- {{ curriculum_year }}
+| Field | Old Value | New Value |
|---|---|---|
| {{ c.field }} | {{ c.old }} | {{ c.new }} |
| Field | Old Value | New Value |
|---|---|---|
| {{ c.field }} | {{ c.old }} | {{ c.new }} |
| Course Code | +{{ course.course_code | course_code_for_year(course.semester, curriculum_year) }} | +Course Title | +{{ course.course_title }} | +||||
|---|---|---|---|---|---|---|---|
| Program | +{{ course.program }} | +Hours per week/ Credit Assigned | +L | +T | +P | +S | +C | +
| {{ course.lecture_hours }} | +{{ course.tutorial_hours }} | +{{ course.practical_hours }} | +{{ course.self_study }} | +{{ course.credits }} | +|||
| Semester | +{{ course.semester }} | +Type of Course | +{{ course.course_type }} | +||||
| AI Tools / Tools / Languages | +{{ course.tools_languages }} | +Desirable Knowledge | +{{ course.desirable_knowledge }} | +||||
| Prelude | +{{ course.prelude }} | +
|---|---|
| Course Objectives: | +
+
|
+
| Course Contents | +
+ {% if course.units %}
+ {% for unit in course.units %}
+
+
+ {% endfor %}
+ {% else %}
+ -
+ {% endif %}
+ {{ unit.title }} +{{ unit.content }} +{{ unit.hours }} Hours + |
+
| Laboratory | +
+
|
+
| Text Book(s): | +
+ {% if course.text_books %}
+
|
+
| Reference Book(s): | +
+
|
+
| Course Outcome | +
+
|
+
P.E.S. University - Curriculum {{ curriculum_year }}
+{{ code }}
{% endfor %}{{ code }}
{% endfor %}| Sl. No. | +Course Code | +Course Title | +Hours per week | +Credits | +AI Tools / Tools / Languages |
+ Course Type | +|||
|---|---|---|---|---|---|---|---|---|---|
| L | +T | +P | +S | +C | +|||||
| {{ row.number }} | ++ | {{ pname }} | ++ | + | + | + | 4 | ++ | Elective Course | +
| Total | +{{ total_cell(totals.lecture, totals.lecture_credit) }} | +{{ totals.tutorial }} | +{{ totals.practical }} | +{{ totals.self_study }} | +{{ totals.credits }} | ++ | + | ||
| + {% if summary_semester == 5 and group == "a" %}Elective-I{% endif %} + {% if summary_semester == 5 and group == "b" %}Elective-II{% endif %} + {% if summary_semester == 6 and group == "a" %}Elective-III{% endif %} + {% if summary_semester == 6 and group == "b" %}Elective-IV{% endif %} + | +
|---|
| ELECTIVES TO BE OPTED FOR SPECIALIZATION | +|||
|---|---|---|---|
| Sl. No. | +Specialization | +{{ "Elective-I" if summary_semester == 5 else "Elective-III" }} | +{{ "Elective-II" if summary_semester == 5 else "Elective-IV" }} | +
Sign in to continue
+Manage refined courses used by preview, editor, and versions.
+| Semester | +Code | +Course | +Credits | +Visible | ++ |
|---|
An Agentic Curriculum Lifecycle Management System for PES University
+ +Syntagma is a FastAPI + static-frontend application that collects course submissions from faculty, refines them into curriculum-ready records with an LLM, renders the entire syllabus as HTML/PDF, lets an assistant agent propose and apply reviewable edits, and preserves named curriculum snapshots (versions).
+ +It is built for a real, fast-changing syllabus: PESU revamps course content nearly every academic year, so nothing about the produced document is hardcoded. Course data, elective categorization, and specialization brackets all live in the database and are rendered dynamically.
+ +Live Demo: syntagma.lonelyguy.tech (preferred) | Backup: pesucurriculum.vercel.app
+ +
+
+
+ | Component | Technology | Purpose |
|---|---|---|
| Backend framework | FastAPI 0.138 + Uvicorn | ASGI server, routing, validation |
| Language | Python 3.12 | Runtime |
| Frontend | Vanilla HTML/CSS/JS | No build step, served as static files |
| Database | Supabase (PostgreSQL) | Persistent storage for all data |
| Cache | Upstash Redis (optional) | Serverless Redis; falls back to in-memory dict |
| LLM provider | OpenRouter | Streaming + tool calling, fallback model retry |
| PDF engine | WeasyPrint | A4 curriculum PDFs from Jinja2 HTML |
| Templating | Jinja2 | Curriculum layout, course pages, diffs |
| HTTP client | httpx | Async requests to OpenRouter and external URLs |
| Spreadsheet parsing | openpyxl | Reading uploaded .xlsx files for text extraction |
| Markdown rendering | marked.js (CDN) | Agent chat message rendering in browser |
| HTML sanitization | DOMPurify (CDN) | Sanitizing agent-generated HTML |
| PDF text extraction | poppler-utils (pdftotext) | Extracting text from uploaded PDF attachments |
| Auth | Supabase Auth (JWT) | Browser-based authentication |
| Error tracking | Sentry SDK (optional) | Production error monitoring |
| Deployment | Docker on HF Spaces | Backend at port 7860 |
| Frontend deploy | Vercel | Static hosting with /api rewrite to backend |
python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+cd backend && fastapi dev app/main.py
+ Server at http://127.0.0.1:8000. API under /api. Frontend served from frontend/.
source .venv/bin/activate
+pytest # 229 tests
+python -m compileall backend/app # also runs in CI
+ | Layer | Location | Responsibility |
|---|---|---|
| Static frontend | frontend/ | Course entry, course management, PDF preview, agentic editor, version history |
| API backend | backend/app/ | FastAPI routes, validation, refinement, previews, drafts, chat, snapshots |
| Services | backend/app/services/ | Business logic, LLM integration, caching, diffing, tool dispatch |
| Persistence | Supabase Postgres | Raw submissions, refined courses, agent drafts, chat history, attachments, curriculum versions |
| Cache | Redis + in-memory | Course lists, version lists, PDFs, with lazy invalidation |
| Rendering | Jinja2 + WeasyPrint | Curriculum summary pages, course detail pages, PDF exports |
| Model provider | OpenRouter | Submission refinement and live-editor chat with tool calls + fallback model retry |
| Auth | Supabase Auth (JWT) | Browser-based authentication with token verification |
| Monitoring | Sentry SDK | Optional production error tracking |
| Path | Responsibility |
|---|---|
| main.py | FastAPI app, CORS, Supabase/.env loading, mounts /api routers and the static frontend |
| api.py | Aggregates all route routers under a single /api router |
| supabase.py | Supabase client + first_row() helper |
| cache.py | Dual cache (Redis + in-memory), lazy invalidation, prefix-based deletion |
| models/submission.py | CourseSubmission (request contract) and parse_course_code() |
| services/deterministic.py | compute_hours, compute_program, compute_course_type from credit category |
| services/refinement.py | The LLM refinement pipeline (refine) |
| services/curriculum.py | Sorting, ordering, version snapshots, draft records, field updates |
| services/diffing.py | JSON diff, protected-field validation, patch apply/merge |
| services/preview.py | build_course_preview, build_specialization_context |
| services/rendering.py | Jinja2 environment, filters, SEMESTER_NAMES global |
| services/agent_tools.py | Agent tool definitions + TOOLS registry (35 tools) + call_tool |
| services/openrouter.py | call() (one-shot), stream_chat() (tool-calling loop), fallback model retry |
| services/schema.py | REQUIRED_TABLES and schema_status() |
| services/errors.py | database_http_exception() |
| services/attachments.py | Text extraction from PDF/DOCX/XLSX/TXT |
| services/books.py | parse_books() textbook parser |
| services/elective_categorization.py | AI-powered elective-to-track matching |
| routes/health.py | GET /api/health/schema |
| routes/submissions.py | POST /api/submissions, POST /api/submissions/{id}/refine |
| routes/preview.py | Course/HTML/PDF preview endpoints (8 endpoints) |
| routes/refined.py | GET/PATCH a single refined course |
| routes/courses.py | List + toggle visibility + soft-delete refined courses |
| routes/agent.py | Draft + document-draft + tool endpoints (13 endpoints) |
| routes/chat.py | Chat sessions, SSE streaming, attachments, system prompt |
| routes/versions.py | Version CRUD, restore, previews, diffs (10 endpoints) |
| routes/auth.py | Token verification, logout |
| templates/jinja_sample.html | Single course + full document renderer + title page |
| templates/jinja_program.html | Program-level title page (large seal) + PEOs/POs |
| templates/jinja_1_to_8.html | Semester summary tables (1-4, 7-8) |
| templates/jinja_sem_5_6.html | Semester 5/6 electives + specialization tables |
| templates/jinja_diff.html | Structured diff renderer for drafts |
| Path | Purpose |
|---|---|
| index.html | Dashboard hub linking to all surfaces |
| form/ | Raw course submission form with course code parsing |
| courses/ | Refined course list with filtering, visibility toggle, soft delete |
| preview/ | Overall or per-semester PDF preview/download |
| live-editor/ | Agentic Editor: course preview, chat assistant, JSON editor, draft review, version restore |
| versions/ | Snapshot list, preview, comparison, editor handoff |
| auth/ | Sign in (Supabase Auth) |
| shared/ | auth-guard.js, supabase-client.js, shared.css, dialog.js |
Syntagma has six surfaces. Each serves a purpose in the curriculum lifecycle.
+ +
+ Supabase Auth with email/password. JWT verified on every API call. The shared/auth-guard.js script redirects unauthenticated users to /auth/.
+ Quick links to all surfaces: Submission Form, Courses, Preview, Live Editor, Versions, and Documentation.
+ +
+ Faculty enter raw course data. Course codes are auto-parsed: semester, department, credits, and course type extracted from the code. Background refinement kicks in immediately after submission. Alternatively, course details can be sent as attachments in the agent chat, where the agent creates a refined course directly using create_refined_course.
+
+
+ Lists all refined courses. Filter by semester or department. Toggle visibility (hidden courses are excluded from previews and PDFs). Delete with confirmation modal.
+ +
+
+
+ Three view modes: single course, full document, or per-semester. PDFs generated via WeasyPrint in PES University's A4 format with letterhead.
+ +
+
+
+ Two-column workspace. Left: preview iframe (single course or full document). Right: tabbed panel.
+| Tab | What it does |
|---|---|
| Agent | AI assistant via SSE streaming. 35 tools. Creates reviewable drafts, never auto-applies. Thread management with rename/delete. File attachments supported. |
| Fields | Raw JSON editor. "Propose Changes" creates a reviewable draft. "Save" directly updates (non-protected fields only). |
| Review | Loads pending drafts (new courses, single-course, multi-course). Shows diff. Apply or reject. |
+
+
+
+ Named snapshots of the entire curriculum. "Current" at top of sidebar represents live state. Preview any snapshot, compare two versions, or restore a previous state. Restore archives absent courses and resets applied drafts.
+| Step | What happens |
|---|---|
| 1 | POST /api/submissions validates payload against CourseSubmission |
| 2 | parse_course_code() derives offering_department, target_department, semester, credit_category |
| 3 | Inserts into submissions, queues background refinement |
| 4 | refine(submission_id) builds deterministic fields, calls OpenRouter, upserts refined_submissions |
| 5 | Course visible in /api/courses, previews, and editor |
Alternative path: course details sent as chat attachments can be used by the agent to create a refined course directly via create_refined_course, bypassing the submission/refinement pipeline.
UE + YY (year) + DEPT (2-letter) + NUMBER (3-digit) + SUFFIX.
| Digit | Meaning |
|---|---|
| Tens | Credits (0/2/4/5) |
| Hundreds | Semester group |
| Suffix A/B | Odd/even semester parity |
parse_course_code() in models/submission.py is the single source of truth.
Computed from credit_category by services/deterministic.py. Protected from casual edits.
| Category | L | T | P | S | C | Course type |
|---|---|---|---|---|---|---|
| 5 | 4 | 0 | 2 | 5 | 5 | Core Course-Lab Integrated |
| 4 | 4 | 0 | 0 | 4 | 4 | Core Course |
| 2 | 2 | 0 | 0 | 2 | 2 | Core Theory |
| 0 | 0 | 0 | 0 | 0 | 0 | Foundation Course |
Protected fields (diffing.py): program, lecture_hours, tutorial_hours, practical_hours, self_study, credits, course_type. Drafts that change them are blocked.
build_course_preview(row) converts a refined_submissions row into a flat dict for templates. services/rendering.py builds the Jinja2 environment.
| Template | Renders |
|---|---|
jinja_program.html | Title page (PES seal, program name, year) and PEOs/POs |
jinja_sample.html | Individual course pages and summary tables |
jinja_1_to_8.html | Semester summary tables (1-4, 7-8) |
jinja_sem_5_6.html | Electives and specialization tables (5/6) |
jinja_diff.html | Structured diffs for agent drafts |
WeasyPrint converts rendered HTML to PDF. PDFs cached 60s. Course lists and version lists cached 180s.
+ +Fully data-driven. No hardcoded specialization brackets.
+| Table | Schema |
|---|---|
specialization_definitions | One row per track: id, semester, letter, name, key, academic_year |
course_specialization_assignments | One row per (course, track): id, refined_id, specialization_id |
refined_submissions.is_elective | Boolean flag |
build_specialization_context() loads tracks and assignments. Template excludes electives from regular tables, splits by code suffix (AA/BA -> group A), renders specialization assignment tables.
Agent tools: define_specialization, list_specializations, assign_elective_to_tracks, remove_elective_from_tracks, categorize_elective.
Tool-calling LLM loop via openrouter.stream_chat. System prompt in chat.py:chat_system_prompt.
| Component | Purpose |
|---|---|
| Curriculum structure context | Semester ranges, course code encoding, credit categories |
| Draft vs. direct creation guidelines | When to draft vs. create directly |
| Desirable knowledge guidance | How to handle prerequisite knowledge fields |
| Agent acknowledgement instruction | Brief text before tool calls |
update_agent_draft tool mention | Modifying existing drafts |
| Tool | Category | Description |
|---|---|---|
get_current_course_json | Read | Full template-ready course JSON |
get_course_codes | Read | Lightweight IDs (refined_id, code, title, semester) |
get_course_syllabus | Read | Units, objectives, course_outcomes |
get_course_textbooks | Read | text_books, reference_books |
get_course_deterministic | Read | Protected fields (read-only context) |
get_course_lab | Read | Lab experiments, tools/languages |
get_course_fields | Read | Arbitrary field subset for one course |
batch_read_courses | Read | Read specific fields from multiple courses in one call |
get_curriculum_json | Read | Full curriculum, optionally by semester |
list_courses | Read | Course IDs/titles, optionally by semester |
get_curriculum_stats | Read | Aggregate statistics (total courses, credits per semester, course type distribution) |
diff_course_json | Read | Compare two course JSONs |
get_course_draft | Read | Read a staged course draft |
get_document_draft | Read | Read a staged document draft |
get_version | Read | Load a curriculum version snapshot with its course list |
diff_versions | Read | Compare two version snapshots (added/removed/changed courses) |
get_course_assignments | Read | Which specialization tracks a course belongs to |
list_specializations | Read | List track definitions |
get_attachment_text | Read | Read uploaded chat attachments |
fetch_url | Read | Fetch external URL content |
web_search | Read | Search the web for external context |
create_course_draft | Write | Propose changes for a single course (reviewable draft) |
update_agent_draft | Write | Merge fields into an existing draft (avoids duplicates) |
create_document_draft | Write | Propose changes for multiple courses (reviewable draft) |
assign_elective_to_tracks | Write | Categorize an elective into specialization tracks |
remove_elective_from_tracks | Write | Remove a course from specialization tracks |
define_specialization | Write | Create a new specialization track |
categorize_elective | Write | AI-powered elective categorization into existing tracks |
update_deterministic_fields | Write | Change protected fields; produces a blocked draft requiring explicit user approval |
create_refined_course | Write | Create new courses directly (for brand-new courses only) |
create_spreadsheet | Generate | Generate CSV or Excel (.xlsx) files from structured row data |
create_report | Generate | Generate markdown or PDF reports |
create_curriculum_version | Generate | Snapshot the current curriculum state |
signal_done | Control | Signal task completion with a summary |
Streamed from POST /chat/sessions/{id}/messages:
| Event | Description |
|---|---|
status | Status updates (model name, tool execution) |
token | Streamed text tokens from the LLM |
tool_call | Agent invoking a tool (name + arguments) |
tool_result | Tool execution result |
draft | A course draft was created |
document_draft | A document draft was created |
refined_course | A new course was created directly |
context_usage | Token usage statistics |
error | Error occurred |
done | Stream complete |
All messages (user, assistant, tool_call, tool_result) saved to chat_messages with role and metadata. Tool calls include function name and arguments; tool results include output. Conversation history persists across page refreshes.
| Validation | Behavior |
|---|---|
desirable_knowledge | Placeholder values (e.g., "None specified") stripped by _create_refined_course |
| Draft-status courses | _create_course_draft rejects drafts, directs agent to create_refined_course |
create_version_snapshot copies every active refined_submissions into finalized_submissions pinned to a curriculum_versions row. restore overwrites current refined data, archives absent courses, writes revision history.
Dual cache layer (services/cache.py):
| Backend | Config | Behavior |
|---|---|---|
| Redis (Upstash, optional) | REDIS_URL | Survives server restarts |
| In-memory dict | Fallback | Survives within a process |
| Prefix | TTL | Purpose |
|---|---|---|
courses_list | 180s | Course list for /api/courses |
course_pdf: | 60s | Individual course PDF bytes |
version_list | 180s | Version list for /api/versions |
Invalidation: lazy (delete-on-write). Prewarm on startup. Debug: set is_cache_disabled = True in cache.py.
OpenRouter client (services/openrouter.py):
| Parameter | Value |
|---|---|
| Retryable statuses | 502, 503 (server errors) |
| Retryable exceptions | httpx.TimeoutException, httpx.TransportError |
| NOT retried | 429 (rate limit) - shows error and stops immediately |
| Backoff | Fibonacci sequence [1, 1, 2, 3, 5, 8, 13] seconds with +/-25% jitter |
| Max retries | 3 per request |
| Retry-After header | Respected when present |
Fallback model:
+| Parameter | Behavior |
|---|---|
OPENROUTER_FALLBACK_MODEL | Set env var to enable (e.g. google/gemma-3-27b-it:free) |
| Trigger | After 3 failed retries on the primary model |
| Scope | Commits to fallback for the rest of the chat thread |
| Notification | Announced via SSE status events so the UI can inform the user |
| Persistence | Once committed, active_model persists as the fallback for all subsequent calls |
429 (rate limit): No retry, no fallback. Error shown in chat and status bar. User must wait and retry manually.
+Chat streaming errors: Surfaced as SSE error events. UI creates error bubble even when no tokens received. Done handler reloads messages from database.
+All paths are prefixed with /api at runtime.
| Endpoint | Method | Purpose |
|---|---|---|
/api/health/schema | GET | Verify required Supabase tables exist. Returns 503 if tables are missing. |
| Endpoint | Method | Purpose |
|---|---|---|
/api/submissions | POST | Store a raw submission, queue background refinement. Body: CourseSubmission. |
/api/submissions/{id}/refine | POST | Manually trigger refinement for an existing submission. |
| Endpoint | Method | Purpose |
|---|---|---|
/api/refined/{refined_id} | GET | Read template-ready course fields. |
/api/refined/{refined_id} | PATCH | Update editable refined fields. Promotes draft-status courses to refined. Body: { fields: dict }. |
| Endpoint | Method | Purpose |
|---|---|---|
/api/courses | GET | List active refined courses (cached 180s). |
/api/courses/{refined_id}/visible | PATCH | Toggle course visibility. Body: { visible: bool }. |
/api/courses/{refined_id} | DELETE | Soft-delete (archive) a course. |
| Endpoint | Method | Purpose |
|---|---|---|
/api/preview/course/{refined_id} | GET | Render one course as HTML. |
/api/preview/course/{refined_id}/pdf | GET | Generate PDF for one course. ?download=true for attachment. |
/api/preview/html | GET | Render full curriculum as HTML. |
/api/preview/pdf | GET | Generate full curriculum PDF. ?download=true for attachment. |
/api/preview/semester/{sem}/pdf | GET | Generate one semester as PDF. |
/api/preview/semester/{sem}/courses | GET | List course IDs for a semester. |
/api/preview/pending-courses | GET | List draft-status (pending) courses. |
/api/preview/courses | GET | List all visible refined course IDs. |
| Endpoint | Method | Purpose |
|---|---|---|
/api/agent/drafts | GET | List 100 most recent agent drafts. |
/api/agent/drafts | POST | Create a single-course draft. Body: { refined_id, fields, reason? }. |
/api/agent/drafts/{draft_id} | GET | Fetch a draft with base/proposed JSON and diff summary. |
/api/agent/drafts/{draft_id}/apply | POST | Apply a proposed draft. Rejects blocked drafts or protected-field changes. |
/api/agent/drafts/{draft_id}/preview | GET | Render draft as HTML. ?diff=true for diff view. |
| Endpoint | Method | Purpose |
|---|---|---|
/api/agent/document-drafts | GET | List 100 most recent document drafts. |
/api/agent/document-drafts | POST | Create a multi-course draft. Body: { courses: [{ refined_id, fields }], reason? }. |
/api/agent/document-drafts/{id} | GET | Fetch document draft with all linked course drafts. |
/api/agent/document-drafts/{id}/apply | POST | Apply all proposed sub-drafts. |
/api/agent/document-drafts/{id}/preview | GET | Render all proposed courses. ?diff=true for diff view. |
| Endpoint | Method | Purpose |
|---|---|---|
/api/agent/tools | GET | List all agent tool schemas (OpenAI function-calling format). |
/api/agent/tools/{tool_name} | POST | Execute a tool directly. Body: { arguments: dict }. |
/api/agent/context-length | GET | Return current model's context window size. |
/api/agent/diff | POST | Compute structured diff between two course JSONs. |
| Endpoint | Method | Purpose |
|---|---|---|
/api/chat/sessions | GET | List active sessions. ?refined_id= or ?document_draft_id= to filter. |
/api/chat/sessions | POST | Create a session. Body: { refined_id?, document_draft_id?, title? }. |
/api/chat/sessions/{id} | PATCH | Rename a session. Body: { title }. |
/api/chat/sessions/{id} | DELETE | Soft-archive a session and its messages. |
/api/chat/sessions/{id}/messages | GET | Fetch all messages in a session. |
/api/chat/sessions/{id}/messages | POST | Send a message and stream response via SSE. Events: status, token, tool_call, tool_result, draft, document_draft, refined_course, context_usage, error, done. |
/api/chat/sessions/{id}/attachments | POST | Upload files (multipart/form-data). 10MB/file, 25MB total. |
/api/chat/sessions/{id}/attachments/{att_id}/download | GET | Download an attachment. |
/api/chat/sessions/{id}/attachments/{att_id}/preview | GET | Preview an attachment inline. |
| Endpoint | Method | Purpose |
|---|---|---|
/api/versions | GET | List all curriculum snapshots. Includes course_count and has_changes. |
/api/versions | POST | Create a snapshot. Returns 409 if no changes since last version. Body: { name, program?, academic_year?, status? }. |
/api/versions/{version_id} | GET | Fetch version with its course list. |
/api/versions/{version_id} | PATCH | Update version metadata. Body: { name?, academic_year?, status? }. |
/api/versions/{version_id} | DELETE | Delete a version and its finalized_submissions. |
/api/versions/{version_id}/restore | POST | Restore a snapshot. Archives absent courses. Resets applied drafts. |
/api/versions/{version_id}/courses/{refined_id} | GET | Fetch a single course snapshot from a version. |
/api/versions/{version_id}/courses/{refined_id}/preview | GET | Render a version course as HTML. |
/api/versions/{version_id}/preview | GET | Render all version courses. ?diff=true for diff vs current. |
/api/versions/{id1}/diff/{id2} | GET | Render side-by-side diff between two versions. |
| Endpoint | Method | Purpose |
|---|---|---|
/api/auth/check | GET | Verify the bearer token is valid. Returns user info or 401. |
/api/auth/logout | POST | Sign out the current user. |
| Parameter | Applies to | Description |
|---|---|---|
curriculum_year | All preview/PDF endpoints | Pin the batch year (e.g. 2025-2026). Falls back to CURRICULUM_YEAR env var. |
download | PDF endpoints | Set true to trigger Content-Disposition attachment. |
diff | Draft/version preview | Set true to render diff view instead of proposed content. |
Total: 49 endpoints across 9 route files.
+Run docs/schema.sql in the Supabase SQL editor. Required tables:
submissions, refined_submissions, curriculum_versions, finalized_submissions, agent_drafts, agent_document_drafts, course_revision_history, chat_sessions, chat_messages, chat_attachments, specialization_definitions, course_specialization_assignments.
submissions.status: pending -> refined
+refined_submissions: draft -> refined -> archived
+agent_drafts: proposed -> applied
+ blocked -> proposed (on user edit)
+chat_sessions: active -> archived
+
+ course_code, course_title, semester, credit_category, program, lecture_hours, tutorial_hours, practical_hours, self_study, credits, course_type, is_elective, visible, units (jsonb), objectives/text_books/... (arrays), status.
visible (default true) controls whether a course renders in preview/PDF output. Toggle it from the course management page. Hidden courses stay in the database and remain editable but are excluded from every rendered document.
Verify with GET /api/health/schema.
refined_submissions (1) ----< (N) agent_drafts
+refined_submissions (1) ----< (N) course_revision_history
+curriculum_versions (1) ----< (N) finalized_submissions
+agent_document_drafts (1) ----< (N) agent_drafts
+chat_sessions (1) ----< (N) chat_messages
+chat_sessions (1) ----< (N) chat_attachments
+specialization_definitions (1) ----< (N) course_specialization_assignments
+refined_submissions (1) ----< (N) course_specialization_assignments
+
+ | Table | Schema |
|---|---|
specialization_definitions | One row per track: id, semester, letter (A/B/C...), name, key (SCC/MIDS/CSCS), academic_year |
course_specialization_assignments | One row per (course, track) membership: id, refined_id, specialization_id |
refined_submissions.is_elective | Boolean flag marking a course as an elective |
| Variable |
|---|
SUPABASE_URL |
SUPABASE_KEY |
OPENROUTER_URL |
OPENROUTER_API_KEY |
OPENROUTER_MODEL |
| Variable | Description |
|---|---|
CURRICULUM_YEAR | The active batch label (e.g. 2025-2026) |
OPENROUTER_FALLBACK_MODEL | Backup model used when the primary model fails after retries (e.g. google/gemma-3-27b-it:free) |
REDIS_URL | Upstash Redis URL for persistent caching (falls back to in-memory) |
SENTRY_DSN, SENTRY_ENVIRONMENT, SENTRY_RELEASE | Error tracking configuration |
The frontend uses the public Supabase anon key directly in shared/supabase-client.js.
Docker image runs uvicorn app.main:app --host 0.0.0.0 --port 7860 (HF Space).
.github/workflows/sync-to-hub.yml syncs main to the HF Space.
Required HF Space secrets:
+| Secret | Description |
|---|---|
SUPABASE_URL, SUPABASE_KEY | Database connection |
OPENROUTER_URL, OPENROUTER_API_KEY, OPENROUTER_MODEL | LLM provider |
OPENROUTER_FALLBACK_MODEL | Recommended fallback model |
REDIS_URL | Optional persistent caching |
SENTRY_DSN, SENTRY_ENVIRONMENT, SENTRY_RELEASE | Optional error tracking |
Static hosting with frontend/vercel.json rewriting /api/* to the backend.
Required Vercel environment variables:
+| Variable | Description |
|---|---|
NEXT_PUBLIC_SUPABASE_URL | Public Supabase URL |
NEXT_PUBLIC_SUPABASE_ANON_KEY | Public Supabase anon key |
Runs on push/PR: checkout, Python 3.12, pip install (cached), apt-get install poppler-utils, pytest, python -m compileall backend/app.
python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+cd backend && fastapi dev app/main.py
+ Server at http://127.0.0.1:8000. API under /api. Frontend served from frontend/.
source .venv/bin/activate
+pytest # all tests
+python -m compileall backend/app # also runs in CI
+ ${escapeHtml(text)}`;
+ } else {
+ const text = await response.text();
+ previewBody.innerHTML = `${escapeHtml(text)}`;
+ }
+ } catch (error) {
+ previewBody.innerHTML = `${escapeHtml(text)}`;
+ const rows = lines.map((line) => {
+ const cells = [];
+ let current = "";
+ let inQuotes = false;
+ for (let i = 0; i < line.length; i++) {
+ const ch = line[i];
+ if (inQuotes) {
+ if (ch === '"' && line[i + 1] === '"') { current += '"'; i++; }
+ else if (ch === '"') inQuotes = false;
+ else current += ch;
+ } else {
+ if (ch === '"') inQuotes = true;
+ else if (ch === "\t" || (isCsv && ch === ",")) { cells.push(current); current = ""; }
+ else current += ch;
+ }
+ }
+ cells.push(current);
+ return cells;
+ });
+ const header = rows.shift();
+ if (!header) return `${escapeHtml(text)}`;
+ let html = '| ${escapeHtml(h)} | `; }); + html += "
|---|
| ${escapeHtml(cell)} | `; }); + html += "