Updated Chatbot and Report Generator

app.py

@@ -1,14 +1,17 @@
 import json
 import os
 import copy
+import sqlite3
+import operator
 import streamlit as st
 from openai import OpenAI
-from datetime import datetime
 from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
 from typing import Annotated, List
 from pydantic import BaseModel, Field
 from typing_extensions import TypedDict, Literal
 from langgraph.graph import StateGraph, START, END
+from langgraph.constants import Send
 
 # Page configuration
 st.set_page_config(layout="wide", page_title="JEE Roadmap Planner")
@@ -18,6 +21,8 @@ if "data" not in st.session_state:
     st.session_state.data = None
 if "data_old" not in st.session_state:
     st.session_state.data_old = None
+if "report_data" not in st.session_state:
+    st.session_state.report_data = None
 if "incomplete_tasks" not in st.session_state:
     st.session_state.incomplete_tasks = None
 if "incomplete_task_list" not in st.session_state:
@@ -39,24 +44,46 @@ page = st.sidebar.radio("Navigation", ["Home", "Roadmap Manager", "Task Analysis
 
 
 # For roadmap chatbot
-import sqlite3
+def get_chapters_and_subtopics():
+    with open("full_roadmap.json", "r") as f:
+        data = json.load(f)
+
+    ch_subt = {
+        "Physics": {},
+        "Chemistry": {},
+        "Maths": {}
+    }
+
+    for day in data["schedule"]:
+        for subject in day['subjects']:
+            sub = ch_subt[subject['name']]
+            for task in subject['tasks']:
+                sub[task['ChapterName']] = []
+
+    for day in data["schedule"]:
+        for subject in day['subjects']:
+            sub = ch_subt[subject['name']]
+            for task in subject['tasks']:
+                sub[task['ChapterName']].append(task['subtopic'])
+
+    return ch_subt
 
 # Function to convert NL query to SQL
 def generate_sql_from_nl(prompt):
     client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
 
-    table_struct = """
-        CREATE TABLE IF NOT EXISTS roadmap (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            day_num INTEGER,
-            date TEXT,
-            subject TEXT,
-            chapter_name TEXT,
-            task_type TEXT,
-            time TEXT,
-            subtopic TEXT
-        )
-    """
+    table_struct = """CREATE TABLE IF NOT EXISTS roadmap (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        day_num INTEGER,
+                        date TEXT, -- [yyyy-mm-dd]
+                        subject TEXT, -- [Physics, Chemistry or Maths]
+                        chapter_name TEXT,
+                        task_type TEXT, -- (Concept Understanding, Question Practice, Revision, Test)
+                        time TEXT, -- formatted like '0.5 hour', '1 hour', '2 Hours', and so on
+                        subtopic TEXT,
+                    )"""
+    
+    ch_subt = get_chapters_and_subtopics()
 
     response = client.chat.completions.create(
         model="gpt-4o-mini",
@@ -66,14 +93,19 @@ def generate_sql_from_nl(prompt):
              create an SQL query to extract the related Information from an sqlite3 database with the table 
              structure: {table_struct}. 
 
-             Note: For the time column, the data is formatted like '0.5 hour', '1 hour', '2 hours' and 
-             so on. So make sure create queries that compare just the numbers within the text.
-
+             Note: 
+             - For the time column, the data is formatted like '0.5 hour', '1 hour', '2 hours' and 
+               so on. So make sure to create queries that compare just the numbers within the text.
+             - If the student mention about any chapters or subtopics, browse through this json file {ch_subt},
+               find the one with the closest match to the users query and use only those exact names of Chapers 
+               and Subtopics present in this file to create SQL the query.
+             
              You will also make sure multiple times that you give an SQL
              Query that adheres to the given table structure, and you Output just the SQL query.
              Do not include anyting else like new line statements, ```sql or any other text. Your output
              is going to be directly fed into a Python script to extract the required information. So,
-             please follow all the given Instructions."""},
+             please follow all the given Instructions.
+             """},
             {"role": "user", "content": f"""Keeping the table structure in mind: {table_struct}, 
              Convert this prompt to an SQL query for the given table: {prompt}. Make sure your
              output is just the SQL query, which can directly be used to extract required content"""}
@@ -82,7 +114,7 @@ def generate_sql_from_nl(prompt):
     return response.choices[0].message.content.strip()
 
 # Function to convert SQL output to natural language
-def generate_nl_from_sql_output(prompt, data):
+def generate_nl_from_sql_output(prompt, query, data):
     client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
 
     response = client.chat.completions.create(
@@ -93,7 +125,8 @@ def generate_nl_from_sql_output(prompt, data):
              You are going to play a very crucial role of a Roadmap Assistant, who helps the student out with whatever query
              they have related to their roadmap, the data required to answer the users query is already extracted 
              from the Roadmap table of a SQLite3 database and given to you here {data}. Analyse the users query deeply and 
-             reply to it with the relevant information from the given data in a supportive manner."""},
+             reply to it with the relevant information from the given data in a supportive manner. If you get empty data
+             as an input, deeply analyze the user's prompt and the sql query and give a suitable reply."""},
             {"role": "user", "content": f"""Answer to this users query using the data given to you, while keeping 
              your role in mind: {prompt}"""}
         ]
@@ -112,11 +145,11 @@ def fetch_data_from_sql(sql_query):
 # Main function for chatbot
 def answer_user_query(prompt):
     initialize_roadmap_db()
-    sql = generate_sql_from_nl(prompt)
-    st.write(sql)
-    rows = fetch_data_from_sql(sql)
+    query = generate_sql_from_nl(prompt)
+    st.write(query)
+    rows = fetch_data_from_sql(query)
     st.write(rows)
-    return generate_nl_from_sql_output(prompt, rows)
+    return generate_nl_from_sql_output(prompt, query, rows)
     
 def initialize_roadmap_db():
     if not os.path.exists("jee_roadmap.db"):
@@ -164,6 +197,7 @@ def initialize_roadmap_db():
             print("✅ Database created and data inserted successfully.")
         except Exception as e:
             print(f"⚠️ Error initializing database: {e}")
+
 # Function to load initial data
 def load_initial_data():
     with st.spinner("Loading roadmap data..."):
@@ -244,34 +278,349 @@ def extract_incomplete_tasks():
         st.session_state.incomplete_task_list = incomplete_task_list
         st.success("Incomplete tasks extracted!")
 
-# Function to generate report
-def generate_report():
-    with st.spinner("Generating performance report using AI..."):
-        previous_day = st.session_state.data["schedule"][0]
-        previous_day_roadmap_str = str(previous_day)
-        
+def generate_sql_for_report(llm, prompt):
+    table_struct = """
+      CREATE TABLE IF NOT EXISTS roadmap (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          day_num INTEGER,
+          date TEXT,
+          subject TEXT,
+          chapter_name TEXT,
+          task_type TEXT,
+          time TEXT,
+          subtopic TEXT,
+          task_completed BOOLEAN,
+          completion_timestamp TEXT
+      )
+    """
+
+    response = llm.invoke(
+        [
+        SystemMessage(content=f"""You are a helper who runs in the background of an AI agent,
+        which helps students for their JEE Preparation. Now your job is to analyze the user's prompt and
+        create an SQL query to extract the related Information from an sqlite3 database with the table
+        structure: {table_struct}.
+
+        Note: For the time column, the data is formatted like '0.5 hour', '1 hour', '2 hours' and
+        so on, it tells the amount of time required to complete that specific task. So make sure
+        to create queries that compare just the numbers within the text. For the task_type column,
+        the data is either of these (Concept Understanding, Question Practice, Revision or Test)
+
+        You will also make sure multiple times that you give an SQL
+        Query that adheres to the given table structure, and you output just the SQL query.
+        Do not include anything else like new line statements, ```sql or any other text. Your output
+        is going to be directly fed into a Python script to extract the required information. So,
+        please follow all the given instructions.
+        Verify multiple times that the SQL query is error free for the SQLite3 format."""),
+        HumanMessage(content=f"""Keeping the table structure in mind: {table_struct},
+        Convert this prompt to an SQL query for the given table: {prompt}. Make sure your
+        output is just the SQL query, which can directly be used to extract required content.""")
+        ]
+    )
+    return response.content.strip()
+
+def get_sql_data_for_report(sql_query):
+    conn = sqlite3.connect("jee_full_roadmap.db")
+    cursor = conn.cursor()
+
+    results = []
+    queries = [q.strip() for q in sql_query.strip().split(';') if q.strip()]
+
+    for query in queries:
+        cursor.execute(query)
+        columns = [desc[0] for desc in cursor.description]
+        rows = cursor.fetchall()
+        results.append({
+            "query": query,
+            "columns": columns,
+            "rows": rows
+        })
+    conn.close()
+
+    return results
+
+def create_db_for_report(roadmap_data):
+    if not os.path.exists("jee_full_roadmap.db"):
         try:
-            client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
-            response = client.chat.completions.create(
-                model="gpt-4o-mini",
-                messages=[
-                    {"role": "system", "content": """You will be given a JEE student's previous_day_roadmap and then you have to create
-                    a completely interactive and useful report for the user. 
-                    The report should include a table for task completion rates and data
-                    The student's study pattern
-                    The student's weaknesses and tips to improve.
-                    Make sure that a task is completed only when the "task_completed" key is true and the "time" key tells about how much
-                    tentative that task can take time
-                    Use markdown formatting.
-                    """},
-                    {"role": "user", "content": f"""Here is the user's previous day roadmap in json : {previous_day_roadmap_str}"""}
-                ]
-            )
-            output = response.choices[0].message.content
-            st.session_state.final_report = output
-            st.success("Report generated successfully!")
+            conn = sqlite3.connect("jee_full_roadmap.db")
+            cursor = conn.cursor()
+
+            cursor.execute("DROP TABLE IF EXISTS roadmap")
+            cursor.execute("""
+                CREATE TABLE roadmap (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    day_num INTEGER,
+                    date TEXT,
+                    subject TEXT,
+                    chapter_name TEXT,
+                    task_type TEXT,
+                    time TEXT,
+                    subtopic TEXT,
+                    task_completed BOOLEAN,
+                    completion_timestamp TEXT
+                )
+            """)
+
+            for day in roadmap_data["schedule"]:
+                date = day["date"]
+                day_num = day["dayNumber"]
+                for subj in day["subjects"]:
+                    subject = subj["name"]
+                    for task in subj["tasks"]:
+                        cursor.execute("""
+                            INSERT INTO roadmap (day_num, date, subject, chapter_name, task_type, time, subtopic, task_completed, completion_timestamp)
+                            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        """, (
+                            day_num,
+                            date,
+                            subject,
+                            task["ChapterName"],
+                            task["type"],
+                            task["time"],
+                            task["subtopic"],
+                            task["task_completed"],
+                            task["completion_timestamp"]
+                        ))
+            conn.commit()
+            conn.close()
+            print("✅ Database created and data inserted successfully.")
         except Exception as e:
-            st.error(f"Error generating report: {e}")
+            print(f"⚠️ Error initializing database: {e}")
+
+# Function to generate report
+llm = ChatOpenAI(model="gpt-4o-mini")
+class Section(BaseModel):
+    name: str = Field(
+        description="Name for this section of the report.",
+    )
+    description: str = Field(
+        description="Brief overview of the main topics and concepts to be covered in this section.",
+    )
+    data_requirements: str = Field(
+        description="Description of the data needed from the roadmap database to write this section.",
+    )
+
+class Sections(BaseModel):
+    sections: List[Section] = Field(
+        description="Sections of the report.",
+    )
+
+planner = llm.with_structured_output(Sections)
+
+class State(TypedDict):
+    sections: list[Section]  # List of report sections
+    completed_sections: Annotated[list, operator.add]  # All workers write to this key in parallel
+    final_report: str  # Final report
+
+# Combined helper-worker state
+class ProcessorState(TypedDict):
+    section: Section
+    completed_sections: Annotated[list, operator.add]
+
+def orchestrator(state: State):
+    """Orchestrator that generates a plan for the report with data requirements"""
+
+    schema = """CREATE TABLE IF NOT EXISTS roadmap (
+                  id INTEGER PRIMARY KEY AUTOINCREMENT,
+                  day_num INTEGER,
+                  date TEXT, -- [yyyy-mm-dd]
+                  subject TEXT, -- (Physics, Chemistry or Maths)
+                  chapter_name TEXT,
+                  task_type TEXT, -- (Concept Understanding, Question Practice, Revision, Test)
+                  time TEXT, -- formatted like '0.5 hour', '1 hour', '2 Hours', and so on -- Tells the amount of time required to finish the task
+                  subtopic TEXT,
+                  task_completed BOOLEAN, -- 0/1 indicates task completion status
+                  completion_timestamp TEXT
+                )"""
+
+    # Generate queries
+    report_sections = planner.invoke(
+        [
+            SystemMessage(content=f"""You are responsible for creating a structured plan for a JEE preparation analysis report.
+
+            Audience: The report is intended primarily for students, but must also be insightful to mentors and parents. 
+            Keep the language motivational and supportive, with actionable insights backed by data.
+
+            Report Format: The report will be composed of exactly 4 concise sections. Your job is to define these sections. Each section must include:
+            - **Name**: A short, descriptive title
+            - **Description**: What the section analyzes and how it helps the student
+            - **Data Requirements**: A plain-English description of what fields and metrics are needed from the roadmap 
+                database whose schema is given here: {schema}
+
+            DO NOT invent new sections or formats. Use exactly the following four section templates and fill in the 
+            descriptions and data requirements precisely.
+
+            ---
+
+            ### Study Time Analysis
+
+            **Description**: Analyze how much total time the student planned to spend vs how much they actually completed, 
+            across different subjects and task types. This will help the student understand where their time is really going.
+
+            **Data Requirements**:
+            - Fields: `subject`, `task_type`, `time`, `task_completed`
+            - Metrics: 
+              - Total planned time → SUM of all `time`
+              - Total actual time → SUM of `time` where `task_completed = 1`
+              - Grouped by both `subject` and `task_type`
+
+            ---
+
+            ### Task Completion Metrics
+
+            **Description**: Measure the student’s consistency and follow-through by looking at completion rates across 
+            subjects and task types.
+
+            **Data Requirements**:
+            - Fields: `subject`, `task_type`, `task_completed`
+            - Metrics:
+              - Total tasks → COUNT of all tasks
+              - Completed tasks → COUNT of tasks where `task_completed = 1`
+              - Completion percentage per subject and task type
+
+            ---
+
+            ### Study Balance Analysis
+
+            **Description**: Evaluate how the student's study time is distributed across task types (e.g., Practice, Revision, Test) 
+            within each subject. This highlights over- or under-emphasis on any category.
+
+            **Data Requirements**:
+            - Fields: `subject`, `task_type`, `time`
+            - Metrics:
+              - SUM of `time` for each (subject, task_type) pair where task_completed = 1
+              - Relative distribution of time per subject to detect imbalance
+
+            ---
+
+            ### Strengths and Areas for Improvement
+
+            **Description**:
+            This section analyzes how the student's effort is distributed — not by estimating how long they spent, 
+            but by combining how many tasks they completed and how much time those completed tasks represent. 
+            This helps identify:
+              - Subjects and task types where the student is showing strong commitment
+              - Areas that may be neglected or inconsistently approached
+
+            **Data Requirements**:
+            - Fields: subject, task_type, task_completed, time
+            - Metrics (filtered where task_completed = 1):
+              - Total Number of completed tasks
+              - Total amount of time spent
+              - Grouped by subject and task_type
+            ---
+
+            Important Constraints:
+            - You must include **all the mentioned fields** in the `data_requirements` — no assumptions
+            - Use only **aggregate metrics** — no need for per-task or per-day analysis
+            - Keep descriptions student-focused, clear, and motivational
+            - Do not alter section names or invent new ones
+            - Do not output anything outside the strict format above
+
+            Your output will be passed into a structured data pipeline. Return only the filled-out section definitions as described above.
+            """),
+            HumanMessage(content="""Use the given table structure of the roadmap and decide all the sections of
+            the report along with what should be in it and the clearly mention all the data thats required for it
+            from the roadmap table"""),
+        ]
+    )
+
+    return {"sections": report_sections.sections}
+
+def processor(state: ProcessorState):
+    """Combined helper and worker - gets data and writes section in one step"""
+
+    section = state['section']
+
+    # HELPER PART: Get data for this section
+    sql_query = generate_sql_for_report(llm, section.data_requirements)
+    rows = get_sql_data_for_report(sql_query)
+    # WORKER PART: Write the section using the data
+    section_result = llm.invoke(
+        [
+            SystemMessage(
+                content=f"""Create a concise, data-driven JEE preparation report section that provides actionable insights for students,
+                parents, and mentors.
+
+                Requirements:
+                1. Begin directly with key metrics and insights - no introductory preamble
+                2. Use specific numbers, percentages, and ratios to quantify performance
+                3. Include concise tables or bullet points for clarity where appropriate
+                4. Highlight patterns related to:
+                   - Task completion rates
+                   - Time allocation efficiency
+                   - Subject/topic focus distribution
+                   - Study consistency patterns
+                5. For each observation, provide a brief actionable recommendation focused on student improvement.
+                6. Use professional but motivational tone appropriate for academic context
+                7. Strictly use Markdown for formatting all the tables and the numbers
+                8. Strictly keep each section very focused and write it under 0 to 50 words
+                9. Verify the formatting of all the tables multiple times to ensure the markdown is correct.
+                10. Check all the numbers and calculations made by you multiple times to ensure accuracy
+
+                Base all analysis strictly on the provided data - avoid assumptions beyond what's explicitly given to you.
+                Don't assume anything else, even a little bit.
+
+                *Important*
+                If you receive an empty data input, understand that the student hasn't done tasks matching the given data description. Also, 
+                know that this report is for the student to improve themselves, and they have no part in making sure the data is logged for
+                this analysis. Deeply analyze the SQL query ->{sql_query} and the data description ->{section.data_requirements} used to  
+                extract the data and figure out why there was no data available in the roadmap, which the student went through and write 
+                the section accordingly.
+                """
+            ),
+            HumanMessage(
+                content=f"""Here is the section name: {section.name} and description: {section.description}
+                Data for writing this section: {rows}"""
+            ),
+        ]
+    )
+
+    # Return completed section
+    return {"completed_sections": [section_result.content]}
+
+def synthesizer(state: State):
+    """Synthesize full report from sections"""
+
+    # List of completed sections
+    completed_sections = state["completed_sections"]
+
+    # Format completed section to str to use as context for final sections
+    completed_report_sections = "\n\n---\n\n".join(completed_sections)
+
+    return  {"final_report": completed_report_sections}
+
+# Assign processors function
+def assign_processors(state: State):
+    """Assign a processor to each section in the plan"""
+    return [Send("processor", {"section": s}) for s in state["sections"]]
+
+def generate_report(full_roadmap):
+    with st.spinner("Generating performance report using AI..."):        
+        # Build workflow
+        workflow_builder = StateGraph(State)
+
+        # Add the nodes
+        workflow_builder.add_node("orchestrator", orchestrator)
+        workflow_builder.add_node("processor", processor)
+        workflow_builder.add_node("synthesizer", synthesizer)
+
+        # Add edges to connect nodes
+        workflow_builder.add_edge(START, "orchestrator")
+        workflow_builder.add_conditional_edges("orchestrator", assign_processors, ["processor"])
+        workflow_builder.add_edge("processor", "synthesizer")
+        workflow_builder.add_edge("synthesizer", END)
+
+        # Compile the workflow
+        workflow = workflow_builder.compile()
+
+        # Initialize database
+        create_db_for_report(full_roadmap)
+
+        # Invoke
+        state = workflow.invoke({})
+
+        st.session_state.final_report = state["final_report"]
 
 # Function to extract available dates
 def extract_available_dates():
@@ -301,7 +650,7 @@ def shift_incomplete_tasks():
     with st.spinner("Optimizing task distribution using evaluator-optimizer approach..."):
         try:
             # Initialize needed components
-            llm = ChatOpenAI(model="gpt-4o-mini")
+            llm = ChatOpenAI(model="gpt-4o-mini", api_key = os.getenv("OPENAI_API_KEY"))
             
             # Schema for structured output to use in evaluation
             class Feedback(BaseModel):
@@ -655,57 +1004,63 @@ elif page == "Roadmap Manager":
 elif page == "Task Analysis":
     st.title("📊 Task Analysis")
     
-    if st.session_state.data is None:
-        st.warning("Please load roadmap data first from the Home page.")
+    choice = st.selectbox("Choose the roadmap to use for building report", ["Four Day Roadmap", "Full Roadmap"])
+    if choice == "Four Day Roadmap":
+        if st.session_state.data is None:
+            st.warning("Please load roadmap data first from the Home page.")
+        st.session_state.report_data = st.session_state.data
+    elif choice == "Full Roadmap":
+        with open("synthesized_full_roadmap.json", "r") as f:
+            st.session_state.report_data = json.load(f)
+
+    st.markdown("### Performance Report")
+    
+    if st.button("🔍 Generate Performance Report"):
+        generate_report(st.session_state.report_data)
+    
+    if st.session_state.final_report:
+        st.markdown(st.session_state.final_report)
     else:
-        st.markdown("### Performance Report")
-        
-        if st.button("🔍 Generate Performance Report"):
-            generate_report()
-        
-        if st.session_state.final_report:
-            st.markdown(st.session_state.final_report)
-        else:
-            st.info("Click the button above to generate your performance report.")
+        st.info("Click the button above to generate your performance report.")
+    
+    # Add visualization options
+    if st.session_state.data:
+        st.subheader("Task Breakdown")
         
-        # Add visualization options
-        if st.session_state.data:
-            st.subheader("Task Breakdown")
+        # Simple task statistics
+        if st.checkbox("Show Task Statistics"):
+            task_count = 0
+            subject_counts = {}
+            type_counts = {}
             
-            # Simple task statistics
-            if st.checkbox("Show Task Statistics"):
-                task_count = 0
-                subject_counts = {}
-                type_counts = {}
-                
-                for day in st.session_state.data["schedule"]:
-                    for subject in day["subjects"]:
-                        subject_name = subject["name"]
-                        if subject_name not in subject_counts:
-                            subject_counts[subject_name] = 0
+            for day in st.session_state.report_data["schedule"]:
+                for subject in day["subjects"]:
+                    subject_name = subject["name"]
+                    if subject_name not in subject_counts:
+                        subject_counts[subject_name] = 0
+                    
+                    for task in subject["tasks"]:
+                        subject_counts[subject_name] += 1
+                        task_count += 1
                         
-                        for task in subject["tasks"]:
-                            subject_counts[subject_name] += 1
-                            task_count += 1
-                            
-                            # Count by task type
-                            task_type = task.get("type", "Unknown")
-                            if task_type not in type_counts:
-                                type_counts[task_type] = 0
-                            type_counts[task_type] += 1
-                
-                st.write(f"Total tasks: {task_count}")
-                
-                # Create charts for data visualization
-                col1, col2 = st.columns(2)
-                
-                with col1:
-                    st.subheader("Subject Distribution")
-                    st.bar_chart(subject_counts)
-                
-                with col2:
-                    st.subheader("Task Type Distribution")
-                    st.bar_chart(type_counts)
+                        # Count by task type
+                        task_type = task.get("type", "Unknown")
+                        if task_type not in type_counts:
+                            type_counts[task_type] = 0
+                        type_counts[task_type] += 1
+            
+            st.write(f"Total tasks: {task_count}")
+            
+            # Create charts for data visualization
+            col1, col2 = st.columns(2)
+            
+            with col1:
+                st.subheader("Subject Distribution")
+                st.bar_chart(subject_counts)
+            
+            with col2:
+                st.subheader("Task Type Distribution")
+                st.bar_chart(type_counts)
 # ---- ROADMAP CHATBOT PAGE ----
 elif page == "Roadmap Chatbot":
     st.title("🤖 Roadmap Chatbot Assistant")