Use JSON schema for course outline; implement structured plan generation; add schema file; update planner to produce JSON; update app to write JSON and doc attachments; update requirements and searcher for PDF extraction and unify dependencies.

This update introduces a JSON schema for course outlines and modifies the planner and app to generate structured course plans that follow this schema. A new file 'course_outline_schema.json' defines the outline structure. The planner now reads this schema and instructs the LLM to output valid JSON matching it. The finalize function writes the JSON outline and a Word document to attachments. Searcher gains PDF extraction support and requirements.txt is updated to include PyPDF2 and unify dependencies.

app.py

@@ -485,15 +485,29 @@ def finalize_and_doc(chat_history, chat_pairs, sources, plan, chat_key):
         chat_history = []
     if sources is None:
         sources = []
-    # Generate the course plan text
+    import json
+    # Generate the course plan as structured JSON using the planner
     try:
-        plan_text = plan_course(chat_history, sources)
+        json_string = plan_course(chat_history, sources)
     except Exception as e:
-        plan_text = (
-            "An error occurred while generating the course outline. Please ensure your API keys are configured.\n"
-            f"(Error: {e})"
+        json_string = (
+            "{\n  \"error\": \"An error occurred while generating the course outline.\",\n"
+            f"  \"details\": \"{str(e).replace('"', '\\"')}\"\n}}"
         )
-    # Create a Word document from the plan text and sources
+    # Attempt to parse the JSON to ensure it is valid; if it fails, wrap as raw string
+    try:
+        parsed = json.loads(json_string)
+    except Exception:
+        parsed = None
+    plan_text = json_string
+    # Write the JSON outline to a file for download
+    json_path = "/tmp/course_outline.json"
+    try:
+        with open(json_path, "w") as jf:
+            jf.write(json_string)
+    except Exception:
+        json_path = None
+    # Create a Word document from the JSON string; we simply embed the JSON as text into the document
     try:
         doc_path = outline_to_docx("Course Outline", plan_text, references=sources)
     except Exception as e:
@@ -506,10 +520,13 @@ def finalize_and_doc(chat_history, chat_pairs, sources, plan, chat_key):
         with open(tmp_path, "w") as f:
             f.write(err_msg)
         doc_path = tmp_path
-    # Record the generated document as an attachment tied to this chat
+    # Record the generated JSON and document as attachments
     if chat_key:
         try:
-            add_attachment(chat_key, doc_path, os.path.basename(doc_path))
+            if json_path:
+                add_attachment(chat_key, json_path, os.path.basename(json_path))
+            if doc_path:
+                add_attachment(chat_key, doc_path, os.path.basename(doc_path))
         except Exception:
             pass
     # Fetch updated attachment list

course_outline_schema.json

@@ -0,0 +1,46 @@
+
+{
+  "title": "",
+  "tagline": "",
+  "description": "",
+  "duration": "",
+  "level": "",
+  "audience": "",
+  "prerequisites": "",
+  "main_outcome": "",
+  "learning_objectives": [],
+  "key_takeaways": [],
+  "skills": [],
+  "seo_keywords": [],
+  "real_world_connections": "",
+  "proof_of_learning": "",
+  "tools": [
+    {
+      "name": "",
+      "description": "",
+      "url": ""
+    }
+  ],
+  "course_plan": [
+    {
+      "module_title": "",
+      "lessons": [
+        {
+          "lesson_title": "",
+          "items": [
+            {
+              "type": "",
+              "title": "",
+              "description": ""
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "capstone_project": "",
+  "readings": [],
+  "recommended_next_steps": [],
+  "references": [],
+  "attachments": []
+}

planner.py

@@ -3,64 +3,110 @@ import openai
 
 
 def plan_course(messages, sources):
-    """Use OpenAI to plan a course based on messages and sources. Tries to handle different OpenAI SDK versions."""
-    # Ensure API key is available
-    # Support alternative secret name COURSECREATOR_API_KEY as a fallback for the OpenAI API key
+    """
+    Generate a structured course outline as a JSON object using the conversation and collected sources.
+
+    This function reads a JSON schema from the repository (``course_outline_schema.json``) and instructs
+    the language model to produce an output that strictly follows the schema. The conversation history
+    (``messages``) and list of resources (``sources``) are provided to the model as context.
+
+    Args:
+        messages (list[dict]): Conversation history with roles and content.
+        sources (list[dict]): List of source dictionaries with "title" and "url" keys.
+
+    Returns:
+        str: A JSON string representing the course outline that matches the schema.
+
+    Raises:
+        RuntimeError: If the OpenAI API call fails.
+        ValueError: If an API key is not provided via environment variables.
+    """
+    # Ensure API key is available (support COURSECREATOR_API_KEY as fallback)
     api_key = os.getenv("OPENAI_API_KEY") or os.getenv("COURSECREATOR_API_KEY")
     if not api_key:
         raise ValueError(
             "An OpenAI API key is required to plan the course (set OPENAI_API_KEY or COURSECREATOR_API_KEY)"
         )
+    # Load the JSON schema from the local file to guide the model
+    schema_path = os.path.join(os.path.dirname(__file__) or ".", "course_outline_schema.json")
+    try:
+        with open(schema_path, "r") as f:
+            schema_content = f.read().strip()
+    except Exception:
+        # If the schema is not found, define a minimal fallback structure
+        schema_content = (
+            '{"title":"","description":"","course_plan":[]}'
+        )
+    # Compose system prompt: instruct the model to output JSON matching the schema and to use
+    # information from the conversation and the provided sources.
     system_prompt = (
-        "You are an expert course planner. Use the conversation and sources to propose a structured plan."
+        "You are an expert course planner. Use the conversation and sources provided to produce a "
+        "detailed course outline. Your response MUST be a valid JSON object that strictly follows "
+        "this schema:\n\n"
+        f"{schema_content}\n\n"
+        "Do not wrap your answer in markdown or include any additional commentary. Only output the JSON."
     )
-    formatted_messages = [{"role": "system", "content": system_prompt}]
+    # Build messages array for the model: include system prompt, conversation, and a description of sources
+    formatted_messages = [
+        {"role": "system", "content": system_prompt},
+    ]
+    # Include the conversation history
     for msg in messages:
         formatted_messages.append(msg)
-    # Use a widely supported default model; older OpenAI SDKs may not recognise newer names
+    # Append sources description if present
+    if sources:
+        # Format sources as a numbered list for the model to reference
+        source_lines = []
+        for i, src in enumerate(sources, start=1):
+            if isinstance(src, dict):
+                t = src.get("title", "")
+                u = src.get("url", "")
+                source_lines.append(f"[{i}] {t} - {u}")
+        source_text = "\n".join(source_lines)
+        formatted_messages.append({"role": "system", "content": f"Sources:\n{source_text}"})
+    # Model configuration
     model = os.getenv("OPENAI_MODEL", "gpt-3.5-turbo")
-    temperature = float(os.getenv("TEMPERATURE", "0.7"))
-    max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "2048"))
-    # Try to call OpenAI using v1-style client if available
+    temperature = float(os.getenv("TEMPERATURE", "0.3"))  # Lower temperature for more deterministic JSON
+    max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "4096"))
     try:
-        # Newer OpenAI Python SDK (>=1.0) exposes `OpenAI` client
+        # Use new OpenAI client if available
         if hasattr(openai, "OpenAI"):
             client = openai.OpenAI(api_key=api_key)
-            # Try with max_tokens; fall back to max_completion_tokens if unsupported
             try:
-                response = client.chat.completions.create(
+                resp = client.chat.completions.create(
                     model=model,
                     messages=formatted_messages,
                     temperature=temperature,
                     max_tokens=max_tokens,
                 )
             except Exception:
-                response = client.chat.completions.create(
+                # Fallback to max_completion_tokens if model requires it
+                resp = client.chat.completions.create(
                     model=model,
                     messages=formatted_messages,
                     temperature=temperature,
                     max_completion_tokens=max_tokens,
                 )
-            plan_text = response.choices[0].message.content
+            content = resp.choices[0].message.content
         else:
             # Legacy OpenAI SDK (<1.0)
             openai.api_key = api_key
             try:
-                response = openai.ChatCompletion.create(
+                resp = openai.ChatCompletion.create(
                     model=model,
                     messages=formatted_messages,
                     temperature=temperature,
                     max_tokens=max_tokens,
                 )
             except Exception:
-                response = openai.ChatCompletion.create(
+                resp = openai.ChatCompletion.create(
                     model=model,
                     messages=formatted_messages,
                     temperature=temperature,
                     max_completion_tokens=max_tokens,
                 )
-            plan_text = response["choices"][0]["message"]["content"]
+            content = resp["choices"][0]["message"]["content"]
     except Exception as e:
-        # Propagate error for caller to handle
         raise RuntimeError(f"OpenAI API error: {e}")
-    return plan_text
+    # The content should be valid JSON. Return as string so the caller can write to file or parse.
+    return content

searcher.py

@@ -38,6 +38,14 @@ def run_web_search(query, num_results=5, domain_filter=""):
 import re
 from typing import List, Dict, Optional
 
+# Additional imports for PDF extraction
+import io
+try:
+    from PyPDF2 import PdfReader  # type: ignore
+except ImportError:
+    # PyPDF2 will be installed via requirements; if missing, pdf extraction will be disabled
+    PdfReader = None
+
 # Import DB helpers from sibling module. Note: db.py resides in the same package directory.
 from db import get_resource, upsert_resource
 
@@ -103,6 +111,36 @@ def fetch_and_extract(url: str, timeout: int = 15) -> Optional[Dict]:
         resp.raise_for_status()
     except Exception:
         return None
+    # If the response is a PDF (by content type or URL), attempt to extract text using PyPDF2
+    content_type = resp.headers.get("Content-Type", "").lower()
+    if (content_type.startswith("application/pdf") or url.lower().endswith(".pdf")) and PdfReader is not None:
+        try:
+            # Read PDF content
+            pdf_stream = io.BytesIO(resp.content)
+            reader = PdfReader(pdf_stream)
+            all_text = ""
+            for page in reader.pages:
+                try:
+                    text = page.extract_text() or ""
+                except Exception:
+                    text = ""
+                all_text += text + "\n"
+            if not all_text.strip():
+                return None
+            excerpt = all_text[:2000]
+            # Use the URL as the title for PDFs
+            title = url
+            # Determine domain
+            try:
+                from urllib.parse import urlparse
+                domain = urlparse(url).netloc
+            except Exception:
+                domain = ""
+            upsert_resource(url, title, domain, excerpt, meta={"length": len(all_text), "pdf": True})
+            return get_resource(url)
+        except Exception:
+            # If PDF extraction fails, continue with HTML extraction
+            pass
     # Parse HTML
     soup = BeautifulSoup(resp.text, "html.parser")
     # Title: fall back to URL if missing