Upload 8 files

src/app_assets.py

@@ -0,0 +1,2 @@
+app_logo = "https://mma.prnewswire.com/media/1458111/Great_Learning_Logo.jpg?p=facebook"
+app_icon = "https://www.google.com/imgres?q=Great%20learning%20images&imgurl=https%3A%2F%2Flookaside.fbsbx.com%2Flookaside%2Fcrawler%2Fmedia%2F%3Fmedia_id%3D100068746724814&imgrefurl=https%3A%2F%2Fwww.facebook.com%2FGreatLearningOfficial%2F&docid=6RoTx3PPo1UrRM&tbnid=57yBHLRcHmvNlM&vet=12ahUKEwinsvHC2rCNAxX6S2cHHbcKB44QM3oECGQQAA..i&w=1388&h=1388&hcb=2&ved=2ahUKEwinsvHC2rCNAxX6S2cHHbcKB44QM3oECGQQAA"

src/app_utils.py

@@ -0,0 +1,99 @@
+import json
+from langchain.schema import SystemMessage, HumanMessage
+from typing import Dict, Any
+from langchain_openai import ChatOpenAI
+import nbformat
+from io import StringIO
+from nbconvert import HTMLExporter
+
+# Setting up LLM
+
+## Create a ChatOpenAI model instance using LangChain
+model = ChatOpenAI(
+    openai_api_base="https://aibe.mygreatlearning.com/openai/v1",
+    openai_api_key="gl-U2FsdGVkX1/DTLQlsmj+RdJjPy3igB9qINuaX940XtJ0CPnGc/5sbBkPKah/C829",
+    model="gpt-4o",
+    streaming=False  # Explicitly disabling streaming
+)
+
+def parse_data_dictionary(input_str: str) -> dict:
+    """
+    Converts a plain text data dictionary into a structured JSON format.
+
+    Args:
+        input_str (str): Data dictionary in plain text format.
+
+    Returns:
+        dict: Structured JSON representation of the data dictionary.
+    """
+    lines = input_str.strip().split("\n")
+    data_dict = {}
+    for line in lines:
+        parts = line.split(":")
+        if len(parts) == 2:
+            key = parts[0].strip()
+            value = parts[1].strip()
+            data_dict[key] = value
+    return data_dict
+
+
+def generate_notebook_plan_text(plan: Dict[str, Any], output_txt_file: str = "notebook_plan.txt") -> str:
+    """
+    Converts a JSON notebook plan into a human-readable text document.
+
+    Args:
+        plan (Dict[str, Any]): The JSON notebook plan.
+        output_txt_file (str): The filename to save the generated text document.
+
+    Returns:
+        str: The generated text content.
+    """
+    # Define the system prompt to guide the LLM's behavior
+    system_prompt = """
+    You are an expert AI curriculum designer. Given a structured JSON notebook plan, your task is to convert it into a clear, readable text document. The document should outline the notebook's structure, including sections and subsections, specifying whether each is a markdown or code block. For code blocks, include the 'code_rules' provided. For markdown blocks, include the 'description'. Format the output with appropriate headings and bullet points for clarity.
+    """
+
+    # Convert the JSON plan to a formatted string
+    plan_str = json.dumps(plan, indent=2)
+
+    # Create the messages for the LLM
+    messages = [
+        SystemMessage(content=system_prompt),
+        HumanMessage(content=f"Here is the JSON notebook plan:\n\n{plan_str}")
+    ]
+
+    # Invoke the LLM with the messages
+    response = model.invoke(messages)
+
+    # Extract the generated text content
+    text_content = response.content
+
+    # Save the generated text to a file
+    with open(output_txt_file, "w", encoding="utf-8") as f:
+        f.write(text_content)
+
+    return text_content
+
+
+def render_notebook_from_json(notebook_json: dict) -> str:
+    """
+    Renders a Jupyter notebook from raw JSON.
+
+    Args:
+        notebook_json (dict): The raw notebook JSON.
+
+    Returns:
+        str: The HTML representation of the notebook.
+    """
+    # Convert the JSON to a notebook node
+    notebook = nbformat.from_dict(notebook_json)
+
+    # Use nbconvert to convert the notebook to HTML
+    html_exporter = HTMLExporter()
+    body, resources = html_exporter.from_notebook_node(notebook)
+
+    return body
+
+
+
+

src/exception.py

@@ -0,0 +1,35 @@
+import sys
+
+def error_message_detail(error, error_detail):
+    """
+    Generate a detailed error message for the NBforge application.
+
+    :param error: The exception instance or error message.
+    :param error_detail: The sys module (for accessing traceback details via sys.exc_info()).
+    :return: A formatted string with the file name and line number where the error occurred.
+    """
+    _, _, exc_tb = error_detail.exc_info()
+    file_name = exc_tb.tb_frame.f_code.co_filename
+
+    error_message = "Error occurred python script name [{0}] line number [{1}] error message [{2}]".format(
+            file_name, exc_tb.tb_lineno, str(error)
+        )
+    return error_message
+
+
+class NBForgeException(Exception):
+    """
+    Custom exception class for handling errors in the NBforge notebook pipeline.
+    """
+    def __init__(self, error_message, error_detail):
+        """
+        Initialize the exception with a detailed error message.
+
+        :param error_message: A description of the error.
+        :param error_detail: Pass the sys module to extract traceback information.
+        """
+        super().__init__(error_message)
+        self.error_message = error_message_detail(error_message, error_detail=error_detail)
+
+    def __str__(self):
+        return self.error_message

src/logger.py

@@ -0,0 +1,18 @@
+import logging
+import os
+from datetime import datetime
+import os
+
+LOG_FILE = f"{datetime.now().strftime('%m_%d_%Y_%H_%M_%S')}.log"
+
+logs_path = os.path.join(os.getcwd(), "logs", LOG_FILE)
+
+os.makedirs(logs_path, exist_ok=True)
+
+LOG_FILE_PATH = os.path.join(logs_path, LOG_FILE)
+
+logging.basicConfig(
+    filename=LOG_FILE_PATH,
+    format="[ %(asctime)s ] %(lineno)d %(name)s - %(levelname)s - %(message)s",
+    level=logging.INFO,
+)

src/prompts.py

@@ -0,0 +1,1340 @@
+Exploratory_Data_Analysis = """
+# Exploratory Data Analysis Task Notebook Structure and Content Guidelines
+
+This outline guides the LLM to structure an EDA task notebook. It's adaptable to **any tabular dataset** for comprehensive Exploratory Data Analysis. The LLM must **THINK, ANALYZE, AND DECIDE** which operations fit the specific case study, dataset, and weekly topics.
+
+## 1. Problem Statement (First-level heading markdown)
+## Business Context (Second-level heading markdown)
+* Write the user's business context as is in a separate markdown cell.
+
+## Objective (Second-level heading markdown)
+* Write the user's objective as is in a separate markdown cell.
+
+## Data Dictionary (Second-level heading markdown)
+* Write the user's data dictionary as is in a separate markdown cell.
+* Enhance column descriptions with necessary details (e.g., units, specific formats).
+
+*(This section will have 6 markdown cells in total).*
+
+---
+
+# 2. Install and import the necessary libraries (First-level heading markdown)
+## Install Dependencies (Second-level heading markdown)
+* **Think and decide** on necessary libraries with specific versions (e.g., `pandas`, `numpy`, `matplotlib`, `seaborn`).
+* **Code Cell:** Install all dependencies in one cell.
+
+## Import Libraries (Second-level heading markdown)
+* Import all required libraries.
+* Group imports with clear comments (e.g., "# Data manipulation", "# Visualization").
+* Add comments for less common libraries.
+* **Code Cell:** Import all libraries in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+---
+
+# 3. Loading the dataset (First-level heading markdown)
+* Datasets are primarily tabular (CSV, Excel).
+* **Think and decide** the best method to load your specific data.
+
+## Read Data from Path (Second-level heading markdown)
+* Read data from the provided path (e.g., data = `pd.read_csv`, `pd.read_excel`).
+* **Code Cell:** Load data in one cell.
+
+## Copy Data to New Variable and Preview (Second-level heading markdown)
+* Copy data to a new variable (e.g., `df = data.copy()`).
+* For multiple datasets, **think and decide** what to do here (to copy into different variables or some other).
+* **Code Cell:** Copy and preview data in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 4. Data Overview (First-level heading markdown)
+* Analyze dataset inputs (type, path, data dictionary, dataset details (rows, columns, dtypes)).
+* **Think, reason, and decide** on operations to show initial data sanity and structure. The below are few you should consider in this section.
+
+## Display the first few rows of the dataset (Second-level heading markdown)
+* **Examples of operations:** `df.head()`, `df.sample()`.
+* If multiple datasets are involved, consider the most appropriate way to display them (e.g., merged or separately) based on the problem statement.
+* **Code Cell:** Display first few rows in one cell.
+
+## Checking the shape of the dataset (Second-level heading markdown)
+* **Examples of operations:** `df.shape`.
+* **Code Cell:** Display shape in one cell.
+
+## Displaying data types (Second-level heading markdown)
+* **Examples of operations:** `df.dtypes`, `df.info()`.
+* **Code Cell:** Display data types in one cell.
+
+## Statistical summary of the dataset (Second-level heading markdown)
+* **Examples of operations:** `df.describe()`, `df.describe(include='object')`.
+* **Code Cell:** Display statistical summary in one cell.
+
+## Missing values (Second-level heading markdown)
+* **Think and decide** on checking and handling missing values.
+* **Examples of operations:** `df.isnull().sum()`, `df.dropna()`, `df.fillna()`.
+* **Code Cell(s):** Display nulls in one cell; handle in separate cells (if needed).
+
+## Duplicate values (Second-level heading markdown)
+* **Think and decide** on checking and handling duplicate values.
+* **Examples of operations:** `df.duplicated().sum()`, `df.drop_duplicates()`.
+* **Code Cell(s):** Display duplicates in one cell; remove in another (if needed).
+
+* Identify data quality issues (e.g., inconsistent entries, wrong data types such as date/time as object, integer into object columns).
+* For each column, **think, analyze, and decide** on operations to convert to the correct format for smooth EDA.
+* Perform operations in separate cells. Ensure the dataset is ready for EDA.
+* **Think and decide** on basic EDA to extract info using `groupby`, `loc`, `iloc`.
+* **Code Cell(s):** Perform type conversions, basic EDA, and other cleanups in separate cells.
+
+*(This section will have 10-15 code cells with their necessary second-level markdown cells. Total: 20-25 cells).*
+* **NOTE:** Do not introduce analysis beyond weekly topics. Ensure data is fully ready for EDA.
+
+---
+
+# 5. Exploratory Data Analysis (First-level heading markdown)
+* **NOTE:** Do not introduce analysis beyond weekly topics.
+* Focus on visualizations to understand data distributions and relationships.
+
+## Univariate analysis (Second-level heading markdown)
+* **Think and decide** on relevant visualizations for individual features.
+* **Examples of operations:** Histograms, box plots, count plots for categorical data.
+* Justify why each plot helps answer business questions.
+* **Code Cell(s):** Each plot in its own cell with proper comments.
+
+## Bivariate Analysis (Second-level heading markdown)
+* **Think and decide** on visualizations exploring relationships between two variables crucial for the problem.
+* **Examples of operations:** Scatter plots, grouped bar charts, line plots, heatmaps for correlations.
+* Justify why each plot surfaces meaningful patterns.
+* **Code Cell(s):** Each plot in its own cell with proper comments.
+
+## Multivariate Analysis (Second-level heading markdown)
+* **Think and decide** if multi-feature interactions are necessary for objectives.
+* **Examples of operations:** Pair plots, 3D scatter plots, advanced conditional plots, dimensionality reduction techniques (if covered).
+* Justify how these uncover deeper insights.
+* **Code Cell(s):** Each plot in its own cell woth proper comments.
+
+*(In overall this 5th section will have around 30 cells minimum including code and markdown cells of these three subsection in this).*
+
+---
+
+# 6. Feature Engineering (First-level heading markdown)
+* For this EDA-focused case study, we won't perform complex transformations here.
+* **Think and decide** which new features could be derived from existing columns that might yield useful insights.
+* **Examples of considerations:** Binning numerical data, encoding categorical variables, creating ratio features, extracting date/time components.
+* Explain the rationale for each proposed feature in the business context.
+* **Code Cell(s):** Implement simple feature derivations if relevant for EDA (e.g., creating age groups from birthdate).
+
+*(This section will have 3 markdown cells and 7 code cells, depending on relevance).*
+---
+
+# 7. Insights & Recommendations (First-level heading markdown)
+* Keep this section empty.
+* We will write insights based on outputs after the notebook is executed.
+
+---
+
+**FINAL NOTE:** This is your high-level reference, in overall the entire notebook will consist of atleast 80 to 90 cells of all sections. You must **ONLY THINK, ANALYZE, AND DECIDE** based on the specific problem statement, dataset, and weekly subtopics. Do NOT go beyond the weekly topics. Every decision and operation must be carefully considered and justified in the context of the EDA problem.
+"""
+
+
+
+Macine_Learning = """
+# ML Task Notebook Structure and Content Guidelines
+
+This outline guides the LLM to structure a Machine Learning task notebook. It's adaptable to **any tabular dataset** and **any ML problem type** (e.g., classification, regression, clustering). The LLM must **THINK, ANALYZE, AND DECIDE** which operations fit the specific case study, dataset, and weekly topics.
+
+# 1. Problem Statement (First-level heading markdown)
+## Business Context (Second-level heading markdown)
+* Write the user’s Business context as is in a separate markdown cell.
+
+## Objective (Second-level heading markdown)
+* Write the user's objective as is in a separate markdown cell.
+
+## Data Dictionary (Second-level heading markdown)
+* Write the user's data dictionary as is in a separate markdown cell.
+* Enhance column descriptions with necessary details (e.g., units for money, distance).
+
+*(This section will have 6 markdown cells in total).*
+
+---
+
+# 2. Install and import the necessary libraries (First-level heading markdown)
+## Install Dependencies (Second-level heading markdown)
+* **Think and decide** on necessary libraries with specific versions (e.g., `pandas`, `numpy`, `matplotlib`, `seaborn`, `scikit-learn`).
+* **Code Cell:** Install all dependencies in one cell.
+
+## Import Libraries (Second-level heading markdown)
+* Import all required libraries.
+* Group imports with clear comments (e.g., "# Data manipulation", "# Visualization", "# Machine Learning Models").
+* Add comments for less common libraries.
+* **Code Cell:** Import all libraries in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 3. Loading the dataset (First-level heading markdown)
+* Datasets are primarily tabular (CSV, Excel).
+* **Think and decide** the best method to load your specific data.
+
+## Read Data from Path (Second-level heading markdown)
+* Read data from the provided path (e.g., `pd.read_csv`).
+* **Code Cell:** Load data in one cell.
+
+## Copy Data to New Variable and Preview (Second-level heading markdown)
+* Copy data to a new variable (e.g., `df_copy = df.copy()`).
+* Display a preview to confirm data integrity (e.g., `df_copy.head()`). For multiple datasets, **think and decide** how to display or merge them.
+* **Code Cell:** Copy and preview data in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 4. Data Overview and Preliminary Checks (First-level heading markdown)
+* Analyze dataset inputs (type, path, data dictionary).
+* **Think, reason, and decide** on operations to show initial data sanity and structure.
+
+## Display the first few rows of the dataset (Second-level heading markdown)
+* **Examples of operations:** `df.head()`, `df.sample()`.
+* **Code Cell:** Display first few rows in one cell.
+
+## Checking the shape of the dataset (Second-level heading markdown)
+* **Examples of operations:** `df.shape`.
+* **Code Cell:** Display shape in one cell.
+
+## Displaying data types (Second-level heading markdown)
+* **Examples of operations:** `df.dtypes`, `df.info()`.
+* **Code Cell:** Display data types in one cell.
+
+## Statistical summary of the dataset (Second-level heading markdown)
+* **Examples of operations:** `df.describe()`, `df.describe(include='object')`.
+* **Code Cell:** Display statistical summary in one cell.
+
+## Missing values (Second-level heading markdown)
+* **Think and decide** on checking and handling missing values.
+* **Examples of operations:** `df.isnull().sum()`, `df.dropna()`, `df.fillna()`.
+* **Code Cell(s):** Display nulls in one cell; handle in separate cells (if needed).
+
+## Duplicate values (Second-level heading markdown)
+* **Think and decide** on checking and handling duplicate values.
+* **Examples of operations:** `df.duplicated().sum()`, `df.drop_duplicates()`.
+* **Code Cell(s):** Display duplicates in one cell; remove in another (if needed).
+
+* Identify data quality issues (e.g., inconsistent entries, wrong data types like date/time as object).
+* For each column, **think, analyze, and decide** on operations to convert them to the correct format for smooth analysis.
+* Perform operations in separate cells. Ensure the dataset is preliminarily ready.
+* **Think and decide** on basic EDA to extract info using `groupby`, `loc`, `iloc`.
+* **Code Cell(s):** Perform type conversions, basic EDA, and other cleanups in separate cells.
+
+*(This section will have 8-10 markdown cells and 10-15 code cells. Total: 18-25 cells).*
+* **NOTE:** Do not introduce analysis beyond weekly topics. Ensure data is fully ready for EDA.
+
+---
+
+# 5. Exploratory Data Analysis (EDA) (First-level heading markdown)
+* **NOTE:** Do not introduce analysis beyond weekly topics.
+* Focus on visualizations to understand data distributions and relationships for ML model design. Aim for 5-8 plots.
+
+## Univariate Analysis (Second-level heading markdown)
+* **Think and decide** on relevant visualizations for individual features.
+* **Examples of operations:** Histograms, box plots (numerical); count plots (categorical).
+* Justify why each plot helps answer business questions.
+* **Code Cell(s):** Each plot in its own cell.
+
+## Bivariate Analysis (Second-level heading markdown)
+* **Think and decide** on visualizations exploring relationships between two variables crucial for the problem.
+* **Examples of operations:** Scatter plots, grouped bar charts, line plots, heatmaps for correlations.
+* Justify why each plot surfaces meaningful patterns.
+* **Code Cell(s):** Each plot in its own cell.
+
+*(This section will have 2 markdown cells and 8 code cells. Total: 10 cells).*
+
+---
+
+# 6. Data Preprocessing (First-level heading markdown)
+* Prepare the data for machine learning models.
+* **Think, analyze, and decide** on preprocessing steps based on dataset characteristics and chosen ML model requirements.
+* Each distinct operation in its own cell.
+
+## Handling Missing Values (Second-level heading markdown)
+* **Think and decide** on the most suitable strategy for treating missing values (e.g., imputation, removal).
+* **Examples of operations:** `SimpleImputer`, `dropna()`, contextual imputation.
+* **Code Cell(s):** Implement the chosen strategy.
+
+## Handling Outliers (Second-level heading markdown)
+* **Think and decide** if outlier detection and treatment are necessary.
+* **Examples of operations:** Capping (e.g., `IQR` method), transformation (e.g., log transform), removal.
+* **Code Cell(s):** Implement the chosen strategy.
+
+## Feature Engineering (Second-level heading markdown)
+* **Think and decide** which new features can be derived from existing columns to improve model performance.
+* **Examples of operations:** Creating interaction terms, polynomial features, datetime components, binning numerical features.
+* **Code Cell(s):** Implement code to derive and add new features.
+
+## Encoding Categorical Variables (Second-level heading markdown)
+* **Think and decide** on the most appropriate encoding scheme for categorical variables.
+* **Examples of operations:** One-hot encoding (`pd.get_dummies`), Label encoding (`LabelEncoder`), Target encoding.
+* **Code Cell(s):** Implement the chosen encoding strategy.
+
+## Feature Scaling (Second-level heading markdown)
+* **Think and decide** whether feature scaling is necessary and the most appropriate method.
+* **Examples of operations:** Standardization (`StandardScaler`), Normalization (`MinMaxScaler`), Robust Scaling (`RobustScaler`).
+* **Code Cell(s):** Apply the appropriate scaling technique.
+
+*(This section will have 5-7 markdown cells and 10 code cells. Total: 10-17 cells).*
+* **NOTE:** Justify each preprocessing step by its relevance to model training and alignment with the problem.
+
+---
+
+# 7. Model Building (First-level heading markdown)
+* Define, train, and optimize your machine learning models.
+* **Think and decide** whether a single model or multiple models are appropriate based on problem and data.
+* Each distinct operation in its own cell.
+
+## Train-Test Split (Second-level heading markdown)
+* **Think and decide** on the splitting strategy for training and testing sets.
+* **Key Considerations:** Split ratios (e.g., 70-30), `random_state`, stratification for classification.
+* **Code Cell:** Implement the train-test split.
+
+## Model Training (Second-level heading markdown)
+* **Think and decide** on the most suitable machine learning model(s) to train.
+* **Examples of models:** Logistic Regression, Decision Tree, Random Forest, SVM, Gradient Boosting (for classification/regression); K-Means (clustering).
+* **Code Cell(s):** Initialize and fit your selected model(s) to the training data. Each model training ideally in its own cell.
+
+## Hyperparameter Tuning (Second-level heading markdown)
+* **Think and decide** if hyperparameter tuning is necessary and if it aligns with weekly topics.
+* **Examples of techniques:** Grid Search (`GridSearchCV`), Random Search (`RandomizedSearchCV`).
+* **Considerations:** Cross-validation within tuning for robust estimates.
+* **Code Cell(s):** Implement hyperparameter tuning if needed.
+
+*(This section will have 4 markdown cells and 2-6 code cells. Total: 6-10 cells).*
+* **NOTE:** All chosen models, splitting strategies, and tuning techniques must be covered in weekly topics. Justify decisions.
+
+---
+
+# 8. Model Evaluation (First-level heading markdown)
+* Rigorously evaluate model performance and select the best one.
+* **Crucially, select metrics that align with your problem's business goal.**
+* Each distinct operation in its own cell.
+
+## Metric Selection and Justification (Second-level heading markdown)
+* **Think and decide** on the most appropriate evaluation metric(s).
+* **Examples of metrics:**
+    * **Regression:** Mean Squared Error (MSE), Mean Absolute Error (MAE), R-squared.
+    * **Classification:** Accuracy, Precision, Recall, F1-score, ROC-AUC.
+* Justify your choice of metrics.
+* **Code Cell:** Display and explain the chosen metrics.
+
+## Model Performance Evaluation (Second-level heading markdown)
+* Evaluate the trained model(s) on the unseen test set.
+* **Examples of visualizations:** Confusion Matrix, ROC Curve, Residual Plots.
+* **Code Cell(s):** Make predictions, calculate metrics, and include relevant visualizations.
+
+## Final Best Model Selection (Second-level heading markdown)
+* If multiple models were trained, compare their performance across all chosen metrics.
+* **Think and decide** on the definitive criteria to select the best performing model.
+* **Code Cell:** Present comparison, identify the best model (and optionally save it).
+
+*(This section will have 4 markdown cells and 3-5 code cells. Total: 7-10 cells).*
+* **NOTE:** All evaluation methods and metrics must be covered in weekly topics.
+
+---
+
+# 9. Insights & Recommendations (First-level heading markdown)
+* Keep this section empty.
+* We will write insights based on outputs after the notebook is executed.
+
+---
+
+**FINAL NOTE:** This is your high-level reference, and in overall the entire notebook will consist of atleast 70 to 80 cells of all sections. You must **ONLY THINK, ANALYZE, AND DECIDE** based on the specific problem statement, dataset information, and the user's specific weekly subtopics. Do NOT go beyond the weekly topics. Every decision and every operation performed must be carefully considered and justified in the context of the specific problem given.
+"""
+
+
+
+Deep_Learning = """
+# Deep Learning Task Notebook Structure and Content Guidelines
+
+This outline guides the LLM to structure a Deep Learning task notebook. It's adaptable to **any problem statement** and **any dataset type** (tabular, image, text, or mixed). The LLM must **THINK, ANALYZE, AND DECIDE** which operations fit the specific case study, dataset, and weekly topics.
+
+## 1. Problem Statement (First-level heading markdown)
+## Business Context (Second-level heading markdown)
+* Write the user’s Business context as is in a markdown cell.
+
+## Objective (Second-level heading markdown)
+* Write the user’s objective as is in a markdown cell.
+
+## Data Dictionary (Second-level heading markdown)
+* Write the user's data dictionary as is in a markdown cell.
+* Enhance column descriptions with necessary details (e.g., units).
+
+*(This section will have 6 markdown cells in total).*
+
+---
+
+# 2. Install and import the necessary libraries (First-level heading markdown)
+## Install Dependencies (Second-level heading markdown)
+* **Think and decide** on necessary libraries with specific versions (e.g., `tensorflow`, `torch`, `keras`, `numpy`, `pandas`, `matplotlib`, `seaborn`, `scikit-learn`).
+* **Code Cell:** Install all dependencies in one cell.
+
+## Import Libraries (Second-level heading markdown)
+* Import all required libraries.
+* Group imports with clear comments (e.g., "# Deep Learning Frameworks", "# Data manipulation", "# Visualization").
+* Add comments for less common libraries.
+* **Code Cell:** Import all libraries in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 3. Loading the dataset (First-level heading markdown)
+* DL datasets are diverse: tabular, image files, text documents, or mixed.
+* **Think and decide** the best method to load your specific data type and source.
+
+## Read Data from Path (Second-level heading markdown)
+* Read data from the provided path.
+* **Examples of operations:** `pd.read_csv` (tabular), `tf.keras.utils.image_dataset_from_directory` or `torchvision.datasets.ImageFolder` (images), custom loading for text files.
+* **Code Cell:** Load data in one cell.
+
+## Copy Data to New Variable and Preview (Second-level heading markdown)
+* Copy data to a new variable (if applicable).
+* Display a preview to confirm data integrity. For multiple datasets, **think and decide** how to display or merge them.
+* **Examples of operations:** `df.head()` (tabular), showing sample image paths/labels (image), text snippets (text).
+* **Code Cell:** Copy and preview data in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 4. Data Overview and Preliminary Checks (First-level heading markdown)
+* Analyze dataset inputs (type, path, data dictionary if applicable).
+* **Think, reason, and decide** on preliminary operations to show data sanity and structure.
+
+## Display Basic Dataset Information (Second-level heading markdown)
+* Display initial information.
+* **Examples of operations:** `df.head()`, `df.info()` (tabular); sample image file paths/dimensions (image); text snippets/lengths (text).
+* **Code Cell:** Display info in one cell.
+
+## Checking Data Characteristics (Second-level heading markdown)
+* **Think and decide** what basic characteristics to check based on data type and problem.
+* **Examples of checks:** `df.shape`, `df.dtypes`, `df.describe()` (tabular); image dimensions/channels, file integrity (image); text length distribution, vocabulary size (text).
+* **Code Cell(s):** Each distinct check in its own cell.
+
+## Preliminary Missing Values and Duplicates (Second-level heading markdown)
+* **Think and decide** how to check for missing values and duplicates relevant to your data type.
+* **Examples of checks:** `df.isnull().sum()`, `df.duplicated().sum()` (tabular); checking for empty/corrupt files (image/text).
+* **Code Cell(s):** Display checks in one cell; handle in separate cells (if straightforward).
+
+* Identify data quality issues (e.g., inconsistent entries, incorrect data types, corrupted files).
+* For each column/sample, **think, analyze, and decide** which operations convert them to the correct format for further steps.
+* Perform operations in separate cells. Ensure the dataset is preliminarily ready.
+* **Code Cell(s):** Perform type conversions and other preliminary cleanups in separate cells.
+
+*(This section will have 5-7 markdown cells and 6-10 code cells. Total: 11-17 cells).*
+* **NOTE:** Focus on aspects relevant to DL preprocessing and model building. Do not introduce analysis beyond weekly topics.
+
+---
+
+# 5. Exploratory Data Analysis (EDA) (First-level heading markdown)
+* **NOTE:** Do not introduce analysis beyond weekly topics.
+* Focus on key visualizations to understand data distributions and relationships for DL model design. Aim for 5-8 plots.
+
+## Univariate Analysis (Second-level heading markdown)
+* **Think and decide** on up to 4 plots/analyses revealing individual feature characteristics.
+* **Examples of operations:** Histograms, box plots (tabular); image dimension/aspect ratio distributions, pixel intensity histograms (image); text/sequence length distributions, word frequency plots (text).
+* Justify why each plot helps answer business questions.
+* **Code Cell(s):** Each plot/analysis in its own cell.
+
+## Bivariate/Multivariate Analysis (Second-level heading markdown)
+* **Think and decide** on up to 4 visualizations showing meaningful patterns or critical relationships between features.
+* **Examples of operations:** Scatter plots, heatmaps (tabular); sample images from different classes (image); word clouds per class (text); relationships between mixed data types.
+* Justify why each plot helps answer business questions.
+* **Code Cell(s):** Each plot/analysis in its own cell.
+
+*(This section will have 2 markdown cells and 8 code cells. Total: 10 cells).*
+
+---
+
+# 6. Data Preprocessing (First-level heading markdown)
+* Transform raw data into a format suitable for Deep Learning models.
+* **Think, analyze, and decide** on preprocessing steps based on data type and chosen DL architecture.
+* Each distinct operation in its own cell.
+
+## Handling Missing Values and Outliers (Second-level heading markdown)
+* **Think and decide** if explicit handling of missing values or outliers is still needed (e.g., for tabular components, corrupted files).
+* **Examples of operations:** Contextual imputation, removing corrupted samples.
+* **Code Cell(s):** Implement the chosen strategy.
+
+## Data Type-Specific Preprocessing (Second-level heading markdown)
+* **Think and decide** on exact transformations for your specific data type(s).
+
+### For Image Data
+* **Think and decide** on transformations.
+* **Examples of operations:** Resizing/Rescaling, Pixel Normalization, Data Augmentation (rotation, flipping, zooming), Channel Manipulation.
+* **Code Cell(s):** Implement these image preprocessing steps.
+
+### For Text Data
+* **Think and decide** on text preprocessing steps.
+* **Examples of operations:** Tokenization, Numericalization, Vocabulary Building, Sequence Padding/Truncation, Embedding Layer preparation.
+* **Code Cell(s):** Implement these text preprocessing steps.
+
+### For Tabular Data (if mixed or standalone)
+* **Think and decide** if further transformations are needed.
+* **Examples of operations:** Feature Scaling (Standardization/Normalization), Encoding Categorical Variables (One-hot, Label encoding).
+* **Code Cell(s):** Implement these tabular data preprocessing steps.
+
+## Data Loading Pipelines (Second-level heading markdown)
+* **Think and decide** on efficient data pipelines for DL frameworks (e.g., `tf.data.Dataset`, `torch.utils.data.Dataset`/`DataLoader`).
+* **Considerations:** Batching, shuffling, caching, prefetching.
+* **Code Cell(s):** Implement your data loading pipeline.
+
+*(This section will have 5-8 markdown cells and 8-15 code cells. Total: 13-23 cells).*
+* **NOTE:** Justify each step by its relevance to the DL architecture and problem.
+
+---
+
+# 7. Model Building (First-level heading markdown)
+* Define and train your Deep Learning models.
+* **Think, analyze, and decide** on the most appropriate DL architecture based on problem, data, and objective.
+* Each distinct operation in its own cell.
+
+## Train-Validation-Test Split (Second-level heading markdown)
+* **Think and decide** on the splitting strategy (e.g., 70-15-15).
+* **Considerations:** `random_state`, stratification for classification.
+* **Code Cell:** Implement data split.
+
+## Model Architecture Definition (Second-level heading markdown)
+* **Think and decide** on the most suitable DL architecture for your data type and problem.
+* **Examples of architectures:** Dense Neural Networks (DNNs), Convolutional Neural Networks (CNNs), Recurrent Neural Networks (RNNs - LSTMs/GRUs).
+* **Considerations:** Number of layers, neurons, activation functions, regularization (Dropout), output layer.
+* **Code Cell:** Define your DL model architecture.
+
+## Compile Model (Second-level heading markdown)
+* **Think and decide** on the appropriate loss function and optimizer.
+* **Examples of choices:** `CategoricalCrossentropy`, `MeanSquaredError`, `Adam` optimizer.
+* **Code Cell:** Compile the model.
+
+## Model Training (Second-level heading markdown)
+* Train the compiled DL model.
+* **Key Parameters:** Epochs, Batch Size, Callbacks (e.g., `EarlyStopping`, `ModelCheckpoint`, `ReduceLROnPlateau`).
+* **Code Cell:** Train the model.
+
+*(This section will have 4 markdown cells and 4-6 code cells. Total: 8-10 cells).*
+* **NOTE:** All choices must align with weekly topics. Justify decisions based on problem and data.
+
+---
+
+# 8. Model Evaluation (First-level heading markdown)
+* Rigorously evaluate model performance.
+* **Crucially, select metrics that align with your problem's business goal.**
+* Each distinct operation in its own cell.
+
+## Metric Selection and Justification (Second-level heading markdown)
+* **Think and decide** on the most appropriate evaluation metric(s).
+* **Examples of metrics:** Accuracy, Precision, Recall, F1-score, ROC-AUC (classification); MSE, MAE, R-squared (regression); Perplexity (language models).
+* Justify your choice of metrics.
+* **Code Cell:** Display and explain metrics.
+
+## Model Performance Evaluation (Second-level heading markdown)
+* Evaluate on the unseen test set.
+* **Examples of operations:** Plotting training/validation loss/metric curves, visualizing sample predictions/outputs (images/text), displaying confusion matrix (classification).
+* **Code Cell(s):** Make predictions and calculate/visualize metrics.
+
+## Final Best Model Selection (Second-level heading markdown)
+* If multiple models/configurations were trained, **think, analyze, and decide** on criteria to select the best.
+* Compare performance across chosen metrics.
+* **Code Cell:** Present comparison and identify the best model (and optionally save it).
+
+*(This section will have 3 markdown cells and 3-5 code cells. Total: 6-8 cells).*
+* **NOTE:** All methods/metrics must be covered in weekly topics.
+
+---
+
+# 9. Insights & Recommendations (First-level heading markdown)
+* Keep this section empty.
+* We will write insights based on outputs after the notebook is executed.
+
+---
+
+**FINAL NOTE:** This is your high-level reference, and in overall the entire notebook will consist of atleast 70 to 80 cells of all sections. You must **ONLY THINK, ANALYZE, AND DECIDE** based on the specific problem statement, dataset information (including its type – tabular, image, text, or mixed), and the user's specific weekly subtopics. Do NOT go beyond the weekly topics. Every decision and every operation performed must be carefully considered and justified in the context of the specific Deep Learning problem given.
+"""
+
+
+Natural_Language_Processing = """
+# Natural Language Processing Task Notebook Structure and Content Guidelines
+
+This outline guides the LLM to structure a Deep Learning task notebook. It's adaptable to **any NLP problem** (e.g., text classification, summarization, NER, translation) and **any text dataset format** (e.g., CSV text columns, raw files, PDF/web extractions). The LLM must **THINK, ANALYZE, AND DECIDE** which operations fit the specific case study, dataset, and weekly topics.
+
+## 1. Problem Statement (First-level heading markdown)
+## Business Context (Second-level heading markdown)
+* Write the user’s Business context as is in a separate markdown cell.
+
+## Objective (Second-level heading markdown)
+* Write the user's objective as is in a separate markdown cell.
+
+## Data Dictionary (Second-level heading markdown)
+* Write the user's data dictionary as is in a markdown cell.
+* Enhance column descriptions with necessary details (e.g., units, specific formats).
+
+*(This section will have 6 markdown cells in total).*
+
+---
+
+# 2. Install and import the necessary libraries (First-level heading markdown)
+## Install Dependencies (Second-level heading markdown)
+* **Think and decide** on necessary libraries with specific versions (e.g., `transformers`, `nltk`, `spacy`, `tensorflow`/`torch`).
+* **Code Cell:** Install all dependencies in one cell.
+
+## Import Libraries (Second-level heading markdown)
+* Import all required libraries.
+* Group imports with clear comments (e.g., "# Data manipulation", "# NLP tools").
+* Add comments for less common libraries.
+* **Code Cell:** Import all libraries in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 3. Loading the dataset (First-level heading markdown)
+* NLP datasets are diverse: text in CSVs, raw `.txt` files, text from PDFs/web pages.
+* **Think and decide** the best method to load your specific data type and source.
+
+## Read Data from Path/Source (Second-level heading markdown)
+* Read data from the provided path/source.
+* **Examples of operations:** `pd.read_csv` (tabular with text), iterating `.txt` files, PDF text extraction, web scraping libraries.
+* **Code Cell:** Load data in one cell.
+
+## Copy Data to New Variable and Preview (Second-level heading markdown)
+* Copy data to a new variable (if applicable).
+* Display a preview to confirm data integrity.
+* **Examples of operations:** `df.head()`, sample text snippets with labels.
+* **Code Cell:** Copy and preview data in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 4. Data Overview and Preliminary Checks (First-level heading markdown)
+* Analyze your text data: number of samples, text lengths, initial vocabulary, special characters, class distribution (if classification).
+* **Think, reason, and decide** on preliminary operations to show initial data sanity and structure.
+
+## Display Basic Dataset Information (Second-level heading markdown)
+* Display initial information.
+* **Examples of operations:** `df.info()` (if tabular), sample raw texts, total sample count.
+* **Code Cell:** Display info in one cell.
+
+## Checking Text Characteristics (Second-level heading markdown)
+* **Think and decide** what basic text characteristics to check based on your problem.
+* **Examples of checks:** Text length distribution (histograms, descriptive stats), top N most frequent raw words, unique character sets, encoding issues.
+* **Code Cell(s):** Each distinct check in its own cell.
+
+## Preliminary Missing Values and Duplicates (Second-level heading markdown)
+* **Think and decide** how to check for and handle missing text entries or duplicate samples.
+* **Examples of checks:** `df.isnull().sum()`, `df.duplicated().sum()` (tabular); checking for empty/corrupt text files.
+* **Code Cell(s):** Display checks in one cell; handle issues in separate cells (if straightforward).
+
+* Identify data quality issues (e.g., inconsistent formatting, irrelevant boilerplate, erroneous characters).
+* **Think, analyze, and decide** which initial cleaning/conversion operations are most suitable.
+* Perform all chosen operations in separate cells. Ensure the dataset is preliminarily ready.
+* **Code Cell(s):** Perform initial cleaning operations in separate cells.
+
+*(This section will have 5-7 markdown cells and 6-10 code cells. Total: 11-17 cells).*
+* **NOTE:** Focus on text-specific insights relevant to preprocessing and modeling.
+
+---
+
+# 5. Exploratory Data Analysis (EDA) (First-level heading markdown)
+* **NOTE:** Do not introduce analysis beyond weekly topics.
+* Focus on key text visualizations to understand textual properties, patterns, and relationships. Aim for 5-8 plots.
+
+## Univariate Analysis (Second-level heading markdown)
+* **Think and decide** on up to 4 plots/analyses revealing individual text feature characteristics.
+* **Examples of operations:** Histograms of text lengths (word/character count), bar charts of most frequent unigrams/bigrams.
+* Justify why each plot helps answer business questions.
+* **Code Cell(s):** Each plot/analysis in its own cell.
+
+## Bivariate/Multivariate Analysis (Second-level heading markdown)
+* **Think and decide** on up to 4 visualizations showing meaningful patterns or critical relationships within the text.
+* **Examples of operations:** Word clouds or bar plots of top N-grams specific to different classes, relationship between text length and target variable.
+* Justify why each plot helps address business questions or inform model design.
+* **Code Cell(s):** Each plot/analysis in its own cell.
+
+*(This section will have 2 markdown cells and 8 code cells. Total: 10 cells).*
+
+---
+
+# 6. Data Preprocessing (First-level heading markdown)
+* Transform raw text data into a numerical format suitable for ML/DL models.
+* **Think, analyze, and decide** on preprocessing steps based on text data, NLP task, and model requirements.
+* Each distinct operation in its own cell.
+
+## Text Extraction and Initial Cleaning (Second-level heading markdown)
+* If needed, **think and decide** on robust steps to extract clean, plain text from complex sources (e.g., PDF, HTML).
+* **Think and decide** on initial cleaning steps.
+* **Examples of operations:** Lowercasing, punctuation handling, number handling, special character removal, URL/HTML tag removal.
+* **Code Cell(s):** Implement chosen text extraction and initial cleaning.
+
+## Text Normalization and Stop Word Removal (Second-level heading markdown)
+* **Think and decide** if normalization (stemming/lemmatization) is beneficial.
+* **Think and decide** if stop word removal is appropriate for your specific NLP task.
+* **Code Cell(s):** Implement chosen normalization and/or stop word removal.
+
+## Tokenization (Second-level heading markdown)
+* Break raw text into meaningful units (tokens).
+* **Think and decide** on the most suitable tokenization strategy (word, sentence, subword, or character-level).
+* **Code Cell(s):** Implement chosen tokenization method.
+
+## Numerical Representation (Vectorization/Embedding) (Second-level heading markdown)
+* Convert textual tokens into numerical representations. This is a key decision.
+* **Think and decide** on the most appropriate strategy based on problem, data, and model.
+* **Examples of operations:** Bag-of-Words (CountVectorizer, TfidfVectorizer), static word embeddings (Word2Vec, GloVe, FastText), or contextual embeddings (from Transformers).
+* **Code Cell(s):** Implement chosen text vectorization or embedding generation/loading.
+
+## Sequence Handling (Second-level heading markdown)
+* For sequence-based DL models, **think and decide** on strategies for varying sequence lengths.
+* **Examples of operations:** Padding, truncation, masking.
+* **Code Cell(s):** Implement sequence padding/truncation and masking if needed.
+
+## Data Loading Pipelines (Second-level heading markdown)
+* **Think and decide** on efficient data pipelines for batching preprocessed data to DL frameworks.
+* **Considerations:** Batching, shuffling, caching, prefetching.
+* **Code Cell(s):** Implement your data loading pipeline.
+
+*(This section will have 6-9 markdown cells and 10-18 code cells. Total: 16-27 cells).*
+* **NOTE:** Justify each preprocessing step by its relevance to the NLP task and model.
+
+---
+
+# 7. Model Building (First-level heading markdown)
+* Define and train your NLP models.
+* **Think, analyze, and decide** on the most appropriate model architecture based on problem, data, and objective.
+* Each distinct operation in its own cell.
+
+## Train-Validation-Test Split (Second-level heading markdown)
+* **Think and decide** on the most appropriate strategy for splitting your processed dataset.
+* **Key Considerations:** Split ratios, `random_state`, stratification (for classification).
+* **Code Cell:** Implement the data split.
+
+## Model Architecture Definition (Second-level heading markdown)
+* **Think and decide** on the most suitable DL architecture for your NLP task.
+* **Examples of architectures:** Recurrent Neural Networks (LSTMs/GRUs), 1D CNNs, Attention Mechanisms, Transformer-based Architectures.
+* **Considerations:** Layers, neurons, activation functions, regularization, output layer.
+* **Code Cell:** Define your NLP model architecture.
+
+## Compile Model (Second-level heading markdown)
+* **Think and decide** on the appropriate loss function and optimizer.
+* **Examples of choices:** `CategoricalCrossentropy`, `BinaryCrossentropy`, `Adam` optimizer.
+* **Code Cell:** Compile the defined NLP model.
+
+## Model Training (Second-level heading markdown)
+* Train your compiled NLP model.
+* **Key Parameters:** Epochs, Batch Size, Callbacks (e.g., `EarlyStopping`, `ModelCheckpoint`, `ReduceLROnPlateau`).
+* **Code Cell:** Train the model.
+
+## Transfer Learning (Second-level heading markdown)
+* **Think, analyze, and decide** if transfer learning is applicable and beneficial for your NLP task.
+* **Examples of operations:**
+    * Using pre-trained word embeddings (Word2Vec, GloVe).
+    * Fine-tuning pre-trained **Hugging Face Transformer models** (e.g., BERT, RoBERTa) for specific tasks.
+    * Using **Sentence Transformers** for semantic similarity.
+    * Employing **prompt-based models** (e.g., GPT-style) for text generation/few-shot learning.
+* If applicable, explain rationale and implementation.
+* **Code Cell(s):** Implement transfer learning.
+
+*(This section will have 5-6 markdown cells and 10-15 code cells. Total: 20-25 cells).*
+* **NOTE:** All choices must align with weekly topics. Justify decisions based on problem and data.
+
+---
+
+# 8. Model Evaluation (First-level heading markdown)
+* Rigorously evaluate model performance.
+* **Crucially, select metrics that align with your NLP problem's business goal.**
+* Each distinct operation in its own cell.
+
+## Metric Selection and Justification (Second-level heading markdown)
+* **Think and decide** on the most appropriate evaluation metric(s).
+* **Examples of metrics:**
+    * **Classification/Tagging:** Accuracy, Precision, Recall, F1-score, ROC-AUC, Classification Report.
+    * **Generation/Summarization/Translation:** BLEU, ROUGE, Perplexity.
+* Justify your choice of metrics.
+* **Code Cell:** Display and explain the chosen metrics.
+
+## Model Performance Evaluation (Second-level heading markdown)
+* Evaluate the trained NLP model(s) on the unseen test set.
+* **Examples of operations:** Plotting training/validation loss/metric curves, visualizing sample predictions/generated text, displaying confusion matrix.
+* **Code Cell(s):** Make predictions and calculate/visualize metrics.
+
+## Final Best Model Selection (Second-level heading markdown)
+* If multiple models/configurations were trained, compare their performance.
+* **Think and decide** on the definitive criteria to select the best performing model.
+* **Code Cell:** Present comparison, identify the best model (and optionally save it).
+
+*(This section will have 4 markdown cells and 4-8 code cells. Total: 8-12 cells).*
+* **NOTE:** All evaluation methods and metrics must be covered in weekly topics.
+
+---
+
+# 9. Insights & Recommendations (First-level heading markdown)
+* Keep this section empty.
+* We will write insights based on outputs after the notebook is executed.
+
+---
+
+**FINAL NOTE:** This is your high-level reference. You must **ONLY THINK, ANALYZE, AND DECIDE** based on the specific problem statement, dataset information (including its type – tabular, image, text, or mixed), and the user's specific weekly subtopics. Do NOT go beyond the weekly topics. Every decision and every operation performed must be carefully considered and justified in the context of the specific Deep Learning problem given.
+"""
+
+Computer_Vision = """
+# Computer Vision Task Notebook Structure and Content Guidelines
+
+This outline guides the LLM to structure a Computer Vision task notebook. It's adaptable to **various CV problems** (e.g., image classification, object detection, image segmentation) and **common CV dataset types** (e.g., directories of image files, images with JSON/XML annotations, TFRecords). The LLM must **THINK, ANALYZE, AND DECIDE** which operations fit the specific case study, dataset, and weekly topics.
+
+## 1. Problem Statement (First-level heading markdown)
+## Business Context (Second-level heading markdown)
+* Write the user’s Business context as is in a markdown cell.
+
+## Objective (Second-level heading markdown)
+* Write the user's objective as is in a markdown cell.
+
+## Data Dictionary (Second-level heading markdown)
+* Write the user's data dictionary as is in a markdown cell.
+* Enhance column/label descriptions with necessary details (e.g., class meanings, annotation formats).
+
+*(This section will have 6 markdown cells in total).*
+
+---
+
+# 2. Install and import the necessary libraries (First-level heading markdown)
+## Install Dependencies (Second-level heading markdown)
+* **Think and decide** on necessary libraries with specific versions (e.g., `tensorflow`/`torch`, `keras`, `opencv-python` (cv2), `scikit-image`, `Pillow`, `tqdm`, `albumentations` for augmentation).
+* **Code Cell:** Install all dependencies in one cell.
+
+## Import Libraries (Second-level heading markdown)
+* Import all required libraries.
+* Group imports with clear comments (e.g., "# DL Frameworks", "# Image Processing", "# Visualization").
+* Add comments for less common libraries.
+* **Code Cell:** Import all libraries in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 3. Loading the dataset (First-level heading markdown)
+* CV datasets consist of image files (e.g., `.jpg`, `.png`), often with accompanying annotation files (e.g., `.json`, `.xml`, mask images), directory or Zip file of directories structure defining classes.
+* **Think and decide** the best method to load your specific image data and annotations.
+
+## Read Data from Path/Source (Second-level heading markdown)
+* Read image data and (if applicable) annotations from the provided path/structure.
+* **Examples of operations:** `tf.keras.utils.image_dataset_from_directory` or `torchvision.datasets.ImageFolder` (classification), custom loops for reading image files and parsing annotation files (detection/segmentation).
+* **Code Cell:** Load data in one cell.
+
+## Copy Data to New Variable and Preview (Second-level heading markdown)
+* Copy data to a new variable (if applicable, e.g., list of image paths, dataframe of annotations).
+* Display a preview to confirm data integrity.
+* **Examples of operations:** Displaying a few sample images, showing image paths and corresponding labels/annotations.
+* **Code Cell:** Copy and preview data in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 4. Data Overview and Preliminary Checks (First-level heading markdown)
+* Analyze your image dataset: total images, typical dimensions, channels, file formats, class distribution, potential corruption, annotation format.
+* **Think, reason, and decide** on preliminary operations to show data sanity and structure.
+
+## Display Basic Dataset Information (Second-level heading markdown)
+* Display initial information about the dataset.
+* **Examples of operations:** Total number of images, number of classes (if classification), paths to a few sample images, basic info on image file types.
+* **Code Cell:** Display info in one cell.
+
+## Checking Image Characteristics (Second-level heading markdown)
+* **Think and decide** what basic image characteristics need to be checked based on your problem.
+* **Examples of checks:** Distribution of image dimensions (width/height/aspect ratio), channel distribution (RGB/Grayscale), check for corrupted/unreadable image files, summary statistics of pixel values (e.g., mean/std across channels).
+* **Code Cell(s):** Each distinct check in its own cell.
+
+## Preliminary Missing/Corrupt Samples and Duplicates (Second-level heading markdown)
+* **Think and decide** how to check for and handle missing/corrupted image files or duplicate images/annotations.
+* **Examples of checks:** Iterating through file paths to catch read errors, hashing images to find duplicates, checking for empty/malformed annotation files.
+* **Code Cell(s):** Display checks in one cell; handle issues in separate cells (if straightforward).
+
+* Identify any data quality issues (e.g., inconsistent image sizes, incorrect labels/annotations, irrelevant images).
+* **Think, analyze, and decide** which initial cleaning/conversion operations are most suitable.
+* Perform all chosen operations in separate cells. Ensure the dataset is preliminarily ready.
+* **Code Cell(s):** Perform initial quality checks and cleanups in separate cells.
+
+*(This section will have 5-7 markdown cells and 6-10 code cells. Total: 11-17 cells).*
+* **NOTE:** Focus on image-specific insights relevant to preprocessing and model building.
+
+---
+
+# 5. Exploratory Data Analysis (EDA) (First-level heading markdown)
+* **NOTE:** Do not introduce analysis beyond weekly topics.
+* Focus on key visualizations to understand image properties, label distributions, and (if applicable) annotation patterns. Aim for 5-8 plots/visualizations.
+
+## Univariate Analysis (Second-level heading markdown)
+* **Think and decide** on up to 4 plots/analyses revealing individual image or label characteristics.
+* **Examples of operations:** Bar chart of class distribution, histograms of image dimensions (width, height), pixel intensity histograms for sample images (e.g., mean pixel value distribution).
+* Justify why each visualization helps answer business questions or inform model design.
+* **Code Cell(s):** Each plot/analysis in its own cell.
+
+## Bivariate/Multivariate Analysis (Second-level heading markdown)
+* **Think and decide** on up to 4 visualizations showing relationships or patterns across multiple dimensions.
+* **Examples of operations:** Displaying sample images from different classes, visualizing bounding boxes/masks on images (for detection/segmentation), average image per class, relationships between image properties and labels.
+* Justify why each visualization helps uncover deeper insights relevant to the CV task.
+* **Code Cell(s):** Each plot/analysis in its own cell.
+
+*(This section will have 2 markdown cells and 8 code cells. Total: 10 cells).*
+
+---
+
+# 6. Data Preprocessing (First-level heading markdown)
+* Transform raw image data into a format suitable for Deep Learning models.
+* **Think, analyze, and decide** on preprocessing steps based on image data characteristics, CV task, and chosen model requirements.
+* Each distinct operation must be in its own separate code cell.
+
+## Image-Specific Transformations (Second-level heading markdown)
+* **Think and decide** on necessary transformations for your specific image data.
+* **Examples of operations:**
+    * **Resizing/Rescaling:** Standardizing image dimensions to match model input.
+    * **Pixel Normalization:** Scaling pixel values (e.g., to [0, 1] or [-1, 1]).
+    * **Channel Manipulation:** Converting to grayscale, ensuring consistent channel count (e.g., 3 channels for RGB).
+    * **Data Augmentation:** Applying transformations like rotation, flipping, zooming, brightness/contrast adjustments to increase dataset size and improve model generalization (crucial for DL in CV).
+* **Code Cell(s):** Implement these image preprocessing steps.
+
+## Annotation Preprocessing (if applicable) (Second-level heading markdown)
+* For object detection or segmentation, **think and decide** on necessary annotation processing.
+* **Examples of operations:**
+    * Parsing raw annotations (XML, JSON) into structured formats.
+    * Resizing/scaling bounding box coordinates or mask dimensions to match resized images.
+    * Converting mask images to appropriate tensor formats.
+    * Handling class IDs and one-hot encoding if needed.
+* **Code Cell(s):** Implement annotation preprocessing steps.
+
+## Data Loading Pipelines (Second-level heading markdown)
+* **Think and decide** on the most efficient way to create data pipelines for loading and batching your preprocessed image data for Deep Learning frameworks.
+* **Key Considerations:** Batching, shuffling, caching, prefetching, parallel processing to optimize training speed.
+* **Code Cell(s):** Implement your data loading pipeline (e.g., `tf.data.Dataset`, `torch.utils.data.Dataset` and `DataLoader`).
+
+*(This section will have 3-5 markdown cells and 10-15 code cells. Total: 15-20 cells).*
+* **NOTE:** Justify each preprocessing step by explaining how it prepares the data for specific DL architectures and aligns with the problem objective.
+
+---
+
+# 7. Model Building (First-level heading markdown)
+* Define and train your Computer Vision models.
+* **Think, analyze, and decide** on the most appropriate Deep Learning architecture(s) based on the CV problem, data, and objective.
+* Each distinct operation in its own cell.
+
+## Train-Validation-Test Split (Second-level heading markdown)
+* **Think and decide** on the most appropriate strategy for splitting your processed dataset into training, validation, and testing sets.
+* **Key Considerations:** Choose appropriate split ratios, use `random_state` for reproducibility, decide if stratification is necessary (especially for imbalanced classification tasks).
+* **Code Cell:** Implement the data split.
+
+## Model Architecture Definition (Second-level heading markdown)
+* **Think and decide** on the most suitable Deep Learning architecture for your CV task.
+* **Examples of architectures:**
+    * **Image Classification:** Convolutional Neural Networks (CNNs), pre-trained models from Keras Applications (e.g., ResNet, VGG, EfficientNet) or PyTorch Models.
+    * **Object Detection:** Architectures like YOLO, Faster R-CNN (often involve pre-trained backbones).
+    * **Image Segmentation:** U-Net, FCN (Fully Convolutional Networks).
+* **Considerations:** Number of layers, activation functions, regularization (Dropout, Batch Normalization), output layer matching the problem type.
+* **Code Cell:** Define your CV model architecture.
+
+## Compile Model (Second-level heading markdown)
+* **Think and decide** on the appropriate loss function and optimizer for your specific CV problem.
+* **Loss Functions:** `CategoricalCrossentropy`, `BinaryCrossentropy` (classification); `MeanSquaredError` (regression); `Dice Loss`, `IoU Loss` (segmentation); specialized losses for object detection.
+* **Optimizers:** `Adam`, `SGD` with momentum, `RMSprop`.
+* **Code Cell:** Compile the defined Deep Learning model.
+
+## Model Training (Second-level heading markdown)
+* Train your compiled Deep Learning model on the training data, using the validation set for monitoring performance.
+* **Key Training Parameters:** Epochs, Batch Size, Callbacks (e.g., `EarlyStopping`, `ModelCheckpoint`, `ReduceLROnPlateau`).
+* **Code Cell:** Write the code to train your Deep Learning model.
+
+*(This section will have 4 markdown cells and 15-20 code cells. Total: 20-25 cells).*
+* **NOTE:** All chosen architectures, splitting strategies, and training parameters must be covered in weekly topics. Justify all decisions based on the specific CV problem and data.
+
+---
+
+# 8. Model Evaluation (First-level heading markdown)
+* Rigorously evaluate the performance of your trained Computer Vision model(s).
+* It is crucial to **deeply analyze the problem statement and objective** to select the correct evaluation metrics that directly align with the business goal.
+* Each distinct evaluation operation must be performed in its own separate code cell.
+
+## Metric Selection and Justification (Second-level heading markdown)
+* **Think and decide** on the most appropriate evaluation metric(s) for your CV task.
+* **Examples of metrics:**
+    * **Image Classification:** Accuracy, Precision, Recall, F1-score (macro/micro/weighted), ROC-AUC.
+    * **Image Segmentation:** Intersection over Union (IoU), Dice Coefficient.
+    * **Object Detection:** Mean Average Precision (mAP), IoU for bounding box accuracy.
+* Justify your choice of metrics.
+* **Code Cell:** Display and explain the chosen metrics.
+
+## Model Performance Evaluation (Second-level heading markdown)
+* Evaluate the trained CV model(s) on the unseen test set using the selected metrics.
+* **Examples of visualizations:**
+    * Plotting training/validation loss/metric curves over epochs.
+    * Displaying confusion matrix for classification.
+    * Visualizing sample predictions with overlaid bounding boxes (detection) or masks (segmentation).
+* **Code Cell(s):** Make predictions on the test set, calculate metrics, and include code for any relevant visualizations.
+
+## Final Best Model Selection (Second-level heading markdown)
+* If multiple models/configurations were trained, **think, analyze, and decide** on criteria to select the best one.
+* Compare performance across chosen metrics.
+* **Code Cell:** Present comparison results clearly and explicitly identify the final best model (and optionally save it).
+
+*(This section will have 4 markdown cells and 3-6 code cells. Total: 7-10 cells).*
+* **NOTE:** All evaluation methods and metrics must be covered in weekly topics.
+
+---
+
+# 9. Insights & Recommendations (First-level heading markdown)
+* Keep this section empty.
+* We will write insights based on outputs after the notebook is executed.
+
+---
+
+**FINAL NOTE:** This is your high-level reference. You must **ONLY THINK, ANALYZE, AND DECIDE** based on the specific problem statement, dataset information (including its type – tabular, image, text, or mixed), and the user's specific weekly subtopics. Do NOT go beyond the weekly topics. Every decision and every operation performed must be carefully considered and justified in the context of the specific Computer Vision problem given.
+"""
+
+
+GenerativeAI_Prompt_Engineering = """
+# Generative AI: Prompt Engineering Task Notebook Structure and Content Guidelines
+
+This outline guides the LLM to structure a Prompt Engineering task notebook. It's designed to adapt to **any task solvable via LLMs** (e.g., content generation, summarization, translation, Q&A, code generation) by focusing on **optimizing LLM inputs**. The LLM must **THINK, ANALYZE, AND DECIDE** which prompt engineering techniques fit the specific case study, LLM capabilities, and weekly topics.
+
+## 1. Problem Statement (First-level heading markdown)
+## Business Context (Second-level heading markdown)
+* Write the user’s Business context as is in a markdown cell.
+
+## Objective (Second-level heading markdown)
+* Write the user's objective as is in a markdown cell.
+
+## Key Prompt Components & Desired Output Characteristics (Second-level heading markdown)
+* This section defines the critical elements that will form your prompts (e.g., `[Instruction]`, `[Context]`, `[Examples]`, `[Constraints]`) and the specific characteristics of the *desired output* from the LLM (e.g., JSON format, summary length, specific tone, accuracy requirements).
+* Provide detailed descriptions for each component and output characteristic, **ensuring they align with the business problem and the nature of the input text data source.**
+
+*(This section will have 6 markdown cells in total).*
+
+---
+
+# 2. Install and import the necessary libraries (First-level heading markdown)
+## Install Dependencies (Second-level heading markdown)
+* **Think and decide** on necessary libraries with specific versions. These will primarily be for LLM API interaction, basic data handling (e.g., for CSV, JSON, plain text files), and potentially evaluation tools. **Ensure selected libraries are within the scope of weekly topics.**
+* **Examples:** `openai`, `google-generativeai`, `anthropic` (for LLM APIs); `pandas`, `numpy` (for handling tabular/structured prompt components or simple evaluations); `nltk`, `rouge` (for potential automated evaluation metrics for specific tasks like summarization).
+* **Code Cell:** Install all dependencies in one cell.
+
+## Import Libraries (Second-level heading markdown)
+* Import all required libraries.
+* Group imports with clear comments (e.g., "# LLM API Clients", "# Data Handling & Utilities", "# Evaluation Tools").
+* Add comments for less common libraries.
+* **Code Cell:** Import all libraries in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 3. LLM API Initialization and Configuration (First-level heading markdown)
+This section sets up the connection to the Large Language Model and defines initial generation parameters.
+
+## LLM API Initialization (Second-level heading markdown)
+* Initialize the client for your chosen LLM API using your API key or authentication method. **This step must align with the LLM APIs taught in weekly topics.**
+* **Examples of operations:** `OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))`, `GenerativeModel(model_name="gemini-pro")`.
+* **Code Cell:** Initialize the LLM client.
+
+## Define Generation Parameters (Second-level heading markdown)
+* **Think, analyze, and decide** on initial settings for parameters that control the LLM's output behavior (e.g., creativity, length). These can be tuned later. **Parameter choices should be relevant to the business problem's desired output and within weekly topic scope.**
+* **Examples of parameters:** `temperature` (controls randomness), `max_output_tokens` (controls response length), `top_p` (nucleus sampling), `top_k` (top-k sampling).
+* **Code Cell:** Define a dictionary or variables for these generation parameters.
+
+*(This section will have 2 markdown cells and 2 code cells).*
+
+---
+
+# 4. Understanding the Task and Raw Prompt Inputs (First-level heading markdown)
+This section focuses on analyzing the underlying raw content or context that will be used to construct your prompts. This is akin to "data overview" but specifically tailored for prompt components derived from any text data source.
+
+## Define/Load Raw Input/Context (Second-level heading markdown)
+* Based on the problem statement and the specific text data source, define the primary raw input or context the LLM will work with.
+* **Think, analyze, and decide** the most appropriate method to load or define this content **based on its source format (e.g., CSV, JSON, plain text file, database query results, or directly typed text) and **within weekly topic capabilities**.
+* **Examples of operations:** Define a string variable for a query, load text from a `.txt` file, read a specific column from a `pandas.DataFrame` (if CSV/structured), parse a JSON object, or extract specific content from a simulated web page/document.
+* **Code Cell:** Define/Load the initial raw input/context from its source.
+
+## Define Few-Shot Examples (Optional) (Second-level heading markdown)
+* If your problem benefits from few-shot learning, define or load a small set of high-quality input-output examples. These examples will be inserted directly into your prompts.
+* **Think, analyze, and decide** the best way to represent these examples **based on your data source and the problem's needs**, **staying within weekly topics**.
+* **Examples of operations:** Define a list of dictionaries where each dictionary is an input-output pair, load from a small JSON or CSV file, or hardcode simple examples.
+* **Code Cell:** Define/Load few-shot examples (if applicable).
+
+## Analysis of Raw Input Characteristics (Second-level heading markdown)
+* Analyze basic characteristics of your raw input components relevant to prompt construction. This analysis should inform your prompt design, considering the specific text data source.
+* **Think, analyze, and decide** what checks are relevant **given the data source (e.g., is it clean text, does it have formatting issues, is it short/long?) and the problem objective, always within weekly topic capabilities**.
+* **Examples of checks:** Character/word count of the primary input, structure of sample few-shot examples, identify any special characters, encoding issues, or specific formatting from the source that might need attention (e.g., HTML tags if scraped, JSON key structure).
+* **Code Cell(s):** Display samples and perform basic characteristic checks relevant to the text data source.
+
+*(This section will have 3 markdown cells and 3-5 code cells).*
+
+---
+
+# 5. Prompt Design Patterns & Construction (First-level heading markdown)
+This is the core "engineering" section, where you systematically design and build your prompts using various patterns. Each pattern's implementation should demonstrate its structure and effect, **always adapting to the specific business problem and the processed raw input from your data source.**
+
+## Baseline Prompt Construction (Second-level heading markdown)
+* **Think, analyze, and decide** on a simple, direct prompt to establish a baseline. This will usually involve clear instructions and incorporating the processed raw input from your data source. **Ensure this is within weekly topic scope.**
+* **Code Cell:** Construct the baseline prompt string or message list, dynamically inserting content from your raw input as needed.
+* **Markdown Cell:** Explain the rationale for this baseline.
+
+## Few-Shot Prompting (Second-level heading markdown)
+* **Think, analyze, and decide** if providing explicit examples in the prompt would guide the LLM's style, format, or reasoning better, **based on the specific business problem and the nature of your raw input/desired output. Ensure this pattern is covered in weekly topics.**
+* **Examples of operations:** Concatenate few-shot examples (from Section 4) into the prompt string or add them as message history in a chat API call. The format of these examples must match the LLM's expectation and the problem's requirements.
+* **Code Cell:** Construct the few-shot prompt.
+* **Markdown Cell:** Explain why few-shot is chosen for this problem and how it's implemented.
+
+## Chain-of-Thought (CoT) / Step-by-Step Prompting (Second-level heading markdown)
+* **Think, analyze, and decide** if breaking down complex tasks into intermediate reasoning steps, or instructing the LLM to "think step-by-step," would improve accuracy or transparency for your business problem. **This must align with weekly topics.**
+* **Examples of operations:** Add phrases like "Let's think step by step," or structure the prompt to elicit intermediate outputs before the final answer, integrating the raw input effectively.
+* **Code Cell:** Construct the CoT prompt.
+* **Markdown Cell:** Explain when CoT is useful and its implementation for your specific task.
+
+## Persona-Based / Role-Playing Prompting (Second-level heading markdown)
+* **Think, analyze, and decide** if assigning a specific role or persona to the LLM (e.g., "You are an expert copywriter," "You are a customer service agent") improves the tone, style, or focus of the output for your business problem. **Ensure this is within weekly topics.**
+* **Examples of operations:** Add a system message or an instruction like "You are an expert [role]" at the beginning of the prompt.
+* **Code Cell:** Construct the persona-based prompt.
+* **Markdown Cell:** Explain the benefit of using a persona in the context of the business problem.
+
+## Output Formatting and Constraints (Second-level heading markdown)
+* **Think, analyze, and decide** how to explicitly instruct the LLM on the desired output format (e.g., JSON, markdown, bullet points) or apply constraints (e.g., maximum length, no repetition). This is crucial for matching the business requirement. **This must align with weekly topics.**
+* **Examples of operations:** Add instructions like "Output in JSON format," "Limit response to 3 sentences," "Do not include introductory phrases."
+* **Code Cell:** Construct the prompt with formatting instructions, dynamically incorporating raw input as needed.
+* **Markdown Cell:** Explain the importance of output constraints for solving the business problem.
+
+*(This section will generally consist of approximately 5-7 markdown cells and 10-15 code cells. Total: 15-20 cells).*
+* **NOTE:** Focus on explaining *why* each pattern is chosen for the problem and its specific implementation, always considering the raw input from the data source and staying within weekly topics.
+
+---
+
+# 6. Generating Outputs with Designed Prompts (First-level heading markdown)
+This section focuses on executing the LLM calls using the prompts designed in the previous step and collecting the generated outputs. Operations here are generally consistent across different text data sources and problems.
+
+## Generating Baseline Output (Second-level heading markdown)
+* Use the baseline prompt and initial generation parameters (from Sections 3 & 5) to get an initial output from the LLM. This serves as a point of comparison for engineered prompts.
+* **Code Cell:** Call the LLM API with the baseline prompt and display the generated output.
+
+## Generating Outputs with Engineered Prompts (Second-level heading markdown)
+* For each significant prompt variation (e.g., few-shot, CoT, persona-based) designed in Section 5, generate the LLM's output. The method of calling the LLM remains consistent, but the prompt content changes.
+* **Code Cell(s):** Call the LLM API for each engineered prompt. Store the generated outputs for later evaluation. Display selected outputs.
+
+*(This section will have 2 markdown cells and 5 code cells).*
+
+---
+
+# 7. Evaluation Strategy & Initial Assessment (First-level heading markdown)
+This section focuses on planning *how* to evaluate the generated outputs and performing an initial assessment of quality. The chosen evaluation methods must align perfectly with the specific business problem, the type of output generated, and the text data source.
+
+## Define Evaluation Criteria and Metrics (Second-level heading markdown)
+* **Think, analyze, and decide** on the most appropriate evaluation methods for your task, aligning directly with the business objective and the nature of the LLM's output (e.g., text, structured data). **Ensure these methods are covered in weekly topics.**
+* **Examples of methods:**
+    * **Qualitative Review:** Human inspection against a defined rubric (e.g., factual accuracy, coherence, relevance, tone, conciseness, adherence to source material).
+    * **Quantitative Metrics (if applicable):** BLEU/ROUGE score (for summarization/translation if a reference is available), semantic similarity scores (using embeddings), custom parsing/regex for structured output, accuracy for classification-like tasks. The choice depends heavily on the output's format and the problem.
+* Justify your choice of metrics/criteria, relating them directly to the business problem's success.
+* **Markdown Cell:** Display and explain the chosen evaluation criteria.
+
+## Initial Performance Assessment (Second-level heading markdown)
+* Perform a preliminary assessment of the generated outputs based on your defined criteria. This involves inspecting the outputs against the original input and desired outcome.
+* **Think, analyze, and decide** on the most effective way to display and initially review the outputs, **considering their format and the evaluation criteria. Stay within weekly topics.**
+* **Examples of operations:** Displaying generated outputs side-by-side with original inputs or expected outputs for quick comparison, manually scoring a few samples based on the rubric, or performing quick checks for structured output.
+* **Code Cell(s):** Implement code to display outputs for review and/or perform initial metric calculations.
+
+*(This section will have 2 markdown cells and 5 code cells).*
+
+---
+
+# 8. Insights & Recommendations (First-level heading markdown)
+* Keep this section empty.
+* We will write insights based on outputs after the notebook is executed.
+
+---
+
+**FINAL NOTE:** This is your high-level reference. You must **ONLY THINK, ANALYZE, AND DECIDE** based on the specific problem statement, the **exact text data source (e.g., CSV, JSON, raw text file)**, the capabilities of the chosen LLM, and the user's specific weekly subtopics. **Do NOT go beyond the weekly topics.** Every decision and every operation performed must be carefully considered and justified in the context of the specific Prompt Engineering problem given.
+"""
+
+
+
+GenerativeAI_Retrieval_Augmented_Generation = """
+# Generative AI: Retrieval Augmented Generation (RAG) Application Notebook Structure and Content Guidelines
+
+This outline guides the LLM to structure a RAG application notebook. It's designed to build a RAG system that can adapt to **any business problem requiring factual, source-grounded answers** and **any text-based data source** as its knowledge base. The LLM must **THINK, ANALYZE, AND DECIDE** on the specific implementation details (e.g., chunking strategy, embedding model, vector store, retrieval method) based on the unique problem, the nature of the knowledge base data, and **strictly within the scope of weekly topics**.
+
+## 1. Problem Statement (First-level heading markdown)
+## Business Context (Second-level heading markdown)
+* Write the user’s Business context as is in a markdown cell. This should clearly state the need for factual, source-grounded information.
+
+## Objective (Second-level heading markdown)
+* Write the user's objective as is in a markdown cell. This should describe the desired outcome of the RAG system (e.g., accurately answer questions about internal policies, summarize research papers with citations).
+
+## Knowledge Base Overview & Desired Output Characteristics (Second-level heading markdown)
+* Define the type of knowledge base data that the RAG system will use (e.g., "a collection of PDF manuals," "JSON product specifications," "plain text articles").
+* Describe the specific characteristics of the *desired output* from the RAG system (e.g., factual accuracy, conciseness, inclusion of sources, specific tone). This should align with the business objective and the nature of the information in the knowledge base.
+
+*(This section will have 6 markdown cells in total).*
+
+---
+
+# 2. Install and import the necessary libraries (First-level heading markdown)
+## Install Dependencies (Second-level heading markdown)
+* **Think, analyze, and decide** on necessary libraries with specific versions. These will include LLM API clients, embedding model clients, vector database libraries, document loaders/parsers (e.g., `pypdf`, `python-docx`), and text processing tools. **All selected libraries must be within the scope of weekly topics.**
+* **Examples:** `openai`, `google-generativeai`, `langchain` (or similar orchestration framework if within scope), `sentence-transformers` (for embeddings), `chromadb` (or `faiss-cpu`, `pinecone-client` depending on weekly topics for vector store), `pypdf`, `python-docx`, `tiktoken` (for token counting/chunking).
+* **Code Cell:** Install all dependencies in one cell.
+
+## Import Libraries (Second-level heading markdown)
+* Import all required libraries.
+* Group imports with clear comments (e.g., "# LLM & Embedding Clients", "# Document Loaders", "# Vector Store", "# Text Processing").
+* Add comments for less common libraries.
+* **Code Cell:** Import all libraries in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 3. LLM & Embedding Model Initialization and Configuration (First-level heading markdown)
+This section sets up the core AI models that power the RAG system: the Large Language Model for generation and the Embedding Model for semantic search.
+
+## LLM Initialization (Second-level heading markdown)
+* Initialize the client for your chosen Large Language Model. **This must align with LLM APIs taught in weekly topics.**
+* **Examples of operations:** `OpenAI(api_key=...)`, `GenerativeModel(model_name="gemini-pro")`.
+* **Code Cell:** Initialize the LLM client.
+
+## Embedding Model Initialization (Second-level heading markdown)
+* Initialize the client or load the model for your chosen Embedding Model. This model converts text into numerical vectors. **The chosen embedding model must be within weekly topics.**
+* **Examples of operations:** `SentenceTransformer('all-MiniLM-L6-v2')`, `OpenAIEmbeddings()`.
+* **Code Cell:** Initialize the Embedding Model.
+
+## Define Generation Parameters (Second-level heading markdown)
+* **Think, analyze, and decide** on initial settings for LLM generation parameters (e.g., `temperature`, `max_output_tokens`). These will influence the style and length of the RAG system's answers. **Parameter choices should be relevant to the business problem's desired output and within weekly topic scope.**
+* **Examples of parameters:** `temperature` (controls creativity), `max_output_tokens` (controls response length).
+* **Code Cell:** Define a dictionary or variables for these generation parameters.
+
+*(This section will have 3 markdown cells and 6 code cells (2 for each sub section)).*
+
+---
+
+# 4. Data Ingestion & Preprocessing for Knowledge Base (First-level heading markdown)
+This crucial section involves loading the raw knowledge base documents, extracting their text content, and performing initial cleaning. The operations **must explicitly adapt to the specific text data source(s)** identified in the problem statement.
+
+## Load Knowledge Base Documents (Second-level heading markdown)
+* Load the documents that will form your RAG knowledge base.
+* **Think, analyze, and decide** the most appropriate method to load these documents **based on their specific format (e.g., PDF, DOCX, TXT, JSON, CSV records, web pages) and their location (local files, cloud storage, database). Ensure this is within weekly topic capabilities.**
+* **Examples of operations:** Using `PyPDFLoader` or `PyMuPDF` for PDFs, reading text files, parsing JSON/XML to extract relevant text fields, loading `pandas.DataFrame` from CSV/database and concatenating text columns.
+* **Code Cell(s):** Implement document loading.
+
+## Extract and Clean Text (Second-level heading markdown)
+* Extract raw text content from the loaded documents and perform initial cleaning steps.
+* **Think, analyze, and decide** on necessary cleaning steps **based on the raw data source's characteristics (e.g., removing boilerplate from HTML, handling OCR errors from PDFs, stripping extra whitespace) and the impact on retrieval. Stay within weekly topics.**
+* **Examples of operations:** Using `.page_content` from document loaders, regex for specific pattern removal, basic string cleaning, lowercasing, handling special characters.
+* **Code Cell(s):** Implement text extraction and cleaning. Display samples of cleaned text.
+
+*(This section will have 2 markdown cells and 8 code cells).*
+
+---
+
+# 5. Knowledge Base Chunking Strategy (First-level heading markdown)
+This section defines how the cleaned text is broken into smaller, semantically meaningful "chunks" for indexing and retrieval. The chunking strategy is highly dependent on the nature of the knowledge base and the type of queries expected.
+
+## Choose and Implement Chunking Strategy (Second-level heading markdown)
+* Implement a strategy to divide the cleaned text into manageable chunks.
+* **Think, analyze, and decide** the optimal chunking method, chunk size, and overlap **based on the length and structure of your source documents (e.g., paragraphs, sentences, fixed token counts), the expected query length, and the semantic meaning of the content. Ensure this is within weekly topics.**
+* **Examples of operations:** `RecursiveCharacterTextSplitter`, fixed-size chunking, sentence splitting.
+* **Code Cell:** Implement the chosen chunking strategy and display samples of chunks.
+
+## Analyze Chunk Characteristics (Second-level heading markdown)
+* Analyze basic properties of the generated text chunks.
+* **Think, analyze, and decide** what characteristics are important to check (e.g., average chunk length, distribution of chunk lengths, presence of key information in chunks) **to ensure effective retrieval for the business problem. Stay within weekly topics.**
+* **Examples of checks:** Plotting chunk length distribution, inspecting a random sample of chunks.
+* **Code Cell(s):** Perform chunk characteristic analysis.
+
+*(This section will have 2 markdown cells and 8 code cells).*
+
+---
+
+# 6. Embedding Generation & Vector Store Indexing (First-level heading markdown)
+This section covers converting text chunks into numerical embeddings and storing them in a searchable vector database.
+
+## Generate Embeddings for Chunks (Second-level heading markdown)
+* Convert each text chunk into a high-dimensional numerical vector (embedding) using the initialized embedding model.
+* **Code Cell:** Generate embeddings for all chunks.
+
+## Initialize and Populate Vector Store (Second-level heading markdown)
+* Initialize your chosen vector store and populate it with the generated embeddings and their corresponding text chunks (metadata).
+* **Think, analyze, and decide** on the most suitable vector store **based on the scale of your knowledge base, performance requirements, and whether an in-memory, local, or cloud-based solution aligns with weekly topics.**
+* **Examples of operations:** `Chroma.from_documents()`, `FAISS.from_documents()`, adding embeddings to a cloud vector database such as Azure AI search.
+* **Code Cell:** Initialize and populate the vector store.
+
+*(This section will have 2 markdown cells and 8 code cells).*
+
+---
+
+# 7. Retrieval Mechanism Design (First-level heading markdown)
+This section focuses on how the RAG system will find the most relevant chunks from the vector store given a user's query.
+
+## Define and Implement Retrieval Strategy (Second-level heading markdown)
+* Design the method for retrieving relevant chunks from the vector store based on a user's query.
+* **Think, analyze, and decide** on the most appropriate retrieval strategy **based on the nature of typical user queries, the structure of your knowledge base, and the techniques covered in weekly topics (e.g., simple similarity search, Maximum Marginal Relevance (MMR), hybrid search).**
+* **Examples of operations:** Using `vectorstore.as_retriever()`, configuring search parameters like `k` (number of top results).
+* **Code Cell:** Implement the retrieval strategy.
+
+## Test Retrieval with Sample Queries (Second-level heading markdown)
+* Test the retrieval mechanism with a few representative sample queries to ensure it's returning relevant chunks.
+* **Think, analyze, and decide** on sample queries that cover various aspects of the business problem and the knowledge base content. **Observe if the top-k retrieved chunks are semantically relevant.**
+* **Code Cell(s):** Perform sample retrievals and display the retrieved chunks and their content.
+
+*(This section will have 2 markdown cells and 5 code cells).*
+
+---
+
+# 8. Augmentation & LLM Generation (First-level heading markdown)
+This section brings together the retrieved context and the LLM to generate the final response.
+
+## Construct Augmented Prompt (Second-level heading markdown)
+* Design how the retrieved context will be integrated into the prompt sent to the LLM, along with the user's original query.
+* **Think, analyze, and decide** on the optimal prompt structure to clearly present the retrieved information to the LLM (e.g., "Use the following context to answer the question: [context] Question: [query]"). **Ensure this prompt engineering technique is within weekly topics.**
+* **Code Cell:** Construct the augmented prompt template.
+
+## Generate RAG Response (Second-level heading markdown)
+* Send the augmented prompt to the LLM and generate the final response.
+* **Examples of operations:** Using `llm.invoke()` or `llm.predict()` with the augmented prompt.
+* **Code Cell:** Generate responses for sample queries using the augmented prompt and display them.
+
+*(This section will have 2 markdown cells and 5 code cells).*
+
+---
+
+# 9. RAG System Evaluation (First-level heading markdown)
+This section outlines how to systematically evaluate the performance of the entire RAG system, considering both retrieval and generation quality. The evaluation strategy must be aligned with the business problem's objectives.
+
+## Define Evaluation Metrics and Strategy (Second-level heading markdown)
+* **Think, analyze, and decide** on comprehensive metrics and a strategy to evaluate the RAG system's performance. This should cover:
+    * **Retrieval Quality:** How relevant are the retrieved chunks? (e.g., precision, recall of retrieved documents/chunks if ground truth exists).
+    * **Generation Quality:** How good is the LLM's answer based on the retrieved context? (e.g., faithfulness to context, answer relevance, conciseness, fluency).
+    * **End-to-End RAG Metrics:** (e.g., RAGAS metrics like Faithfulness, Answer Relevance, Context Recall/Precision if within scope).
+* **Crucially, align all chosen metrics with the business problem's success criteria and **ensure they are covered in weekly topics.**
+* **Markdown Cell:** Describe the chosen evaluation metrics and strategy.
+
+## Perform Evaluation (Second-level heading markdown)
+* Implement the chosen evaluation methods on a set of test queries.
+* **Think, analyze, and decide** how to execute the evaluation (e.g., manual human review of outputs, automated scoring if ground truth is available). **Stay within weekly topics.**
+* **Examples of operations:** Loop through test queries, generate RAG responses, apply scoring functions, present results.
+* **Code Cell(s):** Implement and display evaluation results.
+
+## Identify Areas for Improvement (Second-level heading markdown)
+* Based on the evaluation results, identify specific weaknesses in the RAG system (e.g., poor retrieval for certain query types, hallucinations in generation).
+* **Markdown Cell:** Summarize evaluation findings and highlight areas needing improvement.
+
+*(This section will have 3 markdown cells and 7 code cells).*
+
+---
+
+# 10. Insights & Recommendations (First-level heading markdown)
+* Keep this section empty.
+* We will write insights based on outputs after the notebook is executed.
+
+---
+
+**FINAL NOTE:** This is your high-level reference. You must **ONLY THINK, ANALYZE, AND DECIDE** based on the specific problem statement, the **exact text data source(s) for the knowledge base**, the capabilities of the chosen LLM and embedding model, and the user's specific weekly subtopics. **Do NOT go beyond the weekly topics.** Every decision and every operation performed must be carefully considered and justified in the context of the specific RAG application given.
+"""

src/requirements.txt

@@ -0,0 +1,13 @@
+langchain==0.3.24
+langchain-core==0.3.57
+langchain-openai==0.3.15
+langchain_community
+langchain_experimental
+langgraph
+openai
+tiktoken
+papermill
+streamlit
+pandas 
+numpy
+pydantic

src/workflow_nodes.py

@@ -0,0 +1,1266 @@
+# Standard Libraries
+import os
+import json
+import nbformat
+import shutil
+import sys
+import subprocess
+import re
+from pathlib import Path
+
+from typing import Any, Dict, Optional
+from IPython.display import Image, display
+
+# LangChain and OpenAI related imports
+from langchain_openai import ChatOpenAI  # LLM for invoking prompts
+from langchain.schema import SystemMessage
+#from langchain.chat_models import ChatOpenAI
+
+import papermill as pm
+from datetime import datetime
+
+from typing import Any, Dict
+from typing_extensions import TypedDict
+
+
+from workflow_utils import (
+    extract_json_from_response,
+    get_notebook_blueprint,
+    is_plot_code,
+    stage_dependencies,
+    build_plot_insight_agent_executor)
+
+
+
+# Setting up LLM
+
+## Create a ChatOpenAI model instance using LangChain
+model = ChatOpenAI(
+    openai_api_base="https://aibe.mygreatlearning.com/openai/v1",
+    openai_api_key="gl-U2FsdGVkX1/DTLQlsmj+RdJjPy3igB9qINuaX940XtJ0CPnGc/5sbBkPKah/C829",
+    model="gpt-4o",
+    streaming=False  # Explicitly disabling streaming
+)
+
+
+#**Case Study Generator - Prod**
+# **Interactive Case Study State**
+
+class InteractiveCaseStudyState(TypedDict):
+    """
+    Shared state for the interactive, human-in-the-loop Case Study generation workflow.
+    This state dictionary is passed and updated across all workflow nodes.
+
+    Each node reads from and writes to this state, and human review/enhancer nodes
+    use it to store feedback, approval flags, and intermediate artifacts.
+    """
+
+    # ─── User Inputs ────────────────────────────────────────────────────────────
+
+    domain: str
+    # Domain of the case study (e.g., "ml", "dl", "nlp", "cv", "rag", "genai", etc.)
+
+    topics_and_subtopics: str
+    # User-specified weekly topics and subtopics for notebook content focus.
+
+    problem_statement: str
+    # The core business problem or use case the notebook should solve.
+
+    dataset_type: str
+    # Type of dataset uploaded. Accepted values: "csv", "pdf", "images", "json".
+    # Drives downstream logic for loading, preprocessing, and code generation.
+
+    dataset_file_path: str
+    # Automatically populated backend path to the uploaded dataset
+    # (CSV, PDF folder, image folder, etc.). Not entered by the user.
+
+    data_dictionary: Dict[str, str]
+    # Column→description mapping for structured data.
+    # For PDFs or images, may include folder structure or document descriptions.
+
+    additional_instructions: str
+    # Optional user guidance—preprocessing steps, desired models, notebook style, etc.
+    # If omitted, the LLM uses sensible defaults for the chosen domain.
+
+    # ─── Intermediate Artifacts ────────────────────────────────────────────────
+
+    blueprint: str
+    # Text blueprint selected based on `domain`, used to guide Think-Sections.
+
+    plan: Dict[str, Any]
+    # Notebook outline (JSON) generated by the Think-Sections node.
+
+    raw_notebook: Dict[str, Any]
+    # Notebook skeleton with markdown & code cells but no outputs.
+
+    executed_notebook: Dict[str, Any]
+    # Notebook JSON after Papermill execution—includes real outputs.
+
+    annotated_notebook: Dict[str, Any]
+    # Executed notebook interleaved with Observations cells.
+
+    final_notebook: Dict[str, Any]
+    # Final notebook JSON (execution + insights), ready for download.
+
+    #─── Execution Error Capture ──────────────────────────────────────────────
+
+    execution_error: str
+    #If execution failed, the captured exception text from Papermill.
+
+    #─── Human-in-the-Loop Flags & Feedback ─────────────────────────────────────
+
+    _plan_approved: Optional[bool]
+    # False by default. Set True when the user approves the plan.
+
+    plan_feedback: str
+    # If _plan_approved is False, the user's feedback on how to improve the plan.
+
+    _notebook_approved: Optional[bool]
+    # False by default. Set True when the user approves the code skeleton.
+
+    notebook_feedback: str
+    # If _notebook_approved is False, the user's feedback on the notebook before execution.
+
+    _execution_approved: Optional[bool]
+    # False by default. Set True when the user approves the executed notebook.
+
+    execution_feedback: str
+    # If _execution_approved is False, the user's feedback or notes about runtime errors.
+
+
+
+## **Think-Sections**
+def think_sections_node(state):
+    """
+    Think‑Sections Node:
+    - Reads user inputs including domain, topics, problem, dataset_file_path,
+      data_dictionary, and optional additional_instructions.
+    - Retrieves a domain blueprint via get_notebook_blueprint().
+    - Prompts the LLM to generate a structured JSON-based notebook outline.
+    """
+    import json
+    from langchain.schema import SystemMessage
+
+    # 1. Extract inputs from state
+    domain        = state["domain"]
+    topics        = state["topics_and_subtopics"]
+    problem       = state["problem_statement"]
+    dataset_type  = state["dataset_type"]
+    dataset_path  = state["dataset_file_path"]
+    data_dict     = state.get("data_dictionary", {})
+    instructions  = state.get("additional_instructions", "")
+
+    # 2. Get domain-specific blueprint
+    blueprint = get_notebook_blueprint(domain)
+    state["blueprint"] = blueprint
+
+    # 3. Build the prompt
+    prompt = f"""
+You are a world-class AI curriculum architect and notebook planner.
+
+## Domain Blueprint
+Below is the domain-specific blueprint for **{domain}**:
+{blueprint}
+
+---
+
+## Task
+Design a structured, beginner-friendly, and pedagogically sound **notebook plan** in JSON format.
+This notebook will help users solve a real-world problem using their data and provided topics.
+
+---
+
+## User Inputs
+**Domain**: {domain}
+
+**Topics and Subtopics**:
+{topics}
+
+**Problem Statement**:
+{problem}
+
+**Dataset Type**:
+{dataset_type}
+
+**Dataset Location**:
+{dataset_path}
+
+**Data Dictionary**:
+{json.dumps(data_dict, indent=2)}
+
+**Additional Instructions**:
+{instructions if instructions.strip() else "None provided"}
+
+---
+
+## Output Format (JSON Only)
+Return a JSON object with a top-level key `"sections"` — an **ordered list** of notebook sections.
+Each section must follow the rules below:
+
+### Section (1st-level)
+- Represented as a heading with `#` markdown
+- Keys:
+  - `title`: Section heading
+  - `type`: `"markdown"`
+  - `subsections`: List of second-level content blocks
+
+### Subsection (2nd-level)
+- Represented using `##` markdown
+- Keys:
+  - `title`: Subsection heading
+  - `type`: `"markdown"` or `"code"`
+  - For `markdown`:
+    - `description`: Markdown block with supporting explanation
+  - For `code`:
+    - `code_rules`: List of instructions or rules that must be implemented in code
+---
+
+## Constraints to Follow
+- Use names from `data_dictionary` or inferred data context when referring to columns or fields.
+- Always begin with a section for the **Problem Statement** (with subsections for Business Context and Data Dictionary).
+- Align section structure with domain blueprint and user context.
+- Respect all user-given topics, problem, and instructions.
+- Return **only valid JSON**—no extra markdown or commentary outside the JSON object.
+---
+
+Now generate the structured JSON plan for the notebook.
+""".strip()
+
+    # 4. Invoke LLM and parse
+    response = model.invoke([SystemMessage(content=prompt)])
+    plan = extract_json_from_response(response.content)
+
+    # 5. Update state
+    state["plan"] = plan
+    # state["_plan_approved"] = False
+    # state["plan_feedback"] = ""
+
+    return state
+
+# def think_sections_node(state):
+#     """
+#     Think-Sections Node:
+#     - Reads user inputs including domain, topics, problem, dataset_file_path,
+#       data_dictionary, and optional additional_instructions.
+#     - Retrieves a domain blueprint via get_notebook_blueprint().
+#     - Prompts the LLM to generate a detailed, text-based notebook plan with
+#       granular code cell operations, aiming for 70-80 total cells.
+#     """
+#     import json
+#     from langchain.schema import SystemMessage
+
+#     # 1. Extract inputs from state
+#     domain = state["domain"]
+#     topics = state["topics_and_subtopics"]
+#     problem = state["problem_statement"]
+#     dataset_type = state["dataset_type"]
+#     dataset_path = state["dataset_file_path"]
+#     data_dict = state.get("data_dictionary", {})
+#     instructions = state.get("additional_instructions", "")
+
+#     # 2. Get domain-specific blueprint
+#     # This function would now return the text-based blueprint (e.g., EDA_blueprint_text, ML_blueprint_text)
+#     # as described in our previous discussion, NOT the high-level JSON structure.
+#     # For demonstration, I'm using a placeholder. In a real system, you'd fetch the actual text.
+#     blueprint = get_notebook_blueprint(domain)
+#     state["blueprint"] = blueprint # Store the blueprint for later use if needed
+
+#     # 3. Build the prompt
+#     prompt = f"""
+# You are a world-class AI curriculum architect and notebook planner. Your task is to generate a highly detailed,
+# step-by-step notebook plan for a user, ensuring it's beginner-friendly and pedagogically sound.
+
+# ## Your Goal
+# Your primary goal is to **expand the provided domain blueprint into a complete, operational notebook plan**.
+# This plan must meticulously detail every single operation, especially within code cells,
+# to guide the user through solving their problem. The final plan should aim for a total of **70 to 80 cells**.
+
+# ---
+
+# ## Domain Blueprint for {domain}
+# This is the comprehensive guideline for structuring a notebook in the **{domain}** domain. You must strictly
+# adhere to its sections, subsections, and overall flow. For each section, and particularly for each code operation,
+# you will expand on the general rule to create highly specific, actionable steps.
+
+# {blueprint}
+
+# ---
+
+# ## User Inputs to Contextualize the Plan
+
+# Here are the specific details provided by the user. You must integrate these details to
+# make the plan highly relevant and actionable for their specific case.
+
+# **Domain**: {domain}
+# **Topics and Subtopics**: {topics}
+#     * **CRITICAL**: Only include techniques and operations that fall under these specified weekly topics.
+#       Do NOT go beyond this scope.
+
+# **Problem Statement**: {problem}
+#     * **CRITICAL**: Every step and explanation in the plan must directly contribute to addressing this business problem.
+
+# **Dataset Type**: {dataset_type}
+# **Dataset Location**: {dataset_path}
+# **Data Dictionary**: {json.dumps(data_dict, indent=2)}
+#     * **CRITICAL**: Refer to specific columns/fields from the Data Dictionary and dataset details
+#       (e.g., shape, columns, types) when describing operations. Tailor operations to the specific
+#       characteristics of this dataset.
+
+# **Additional Instructions**: {instructions if instructions.strip() else "None provided"}
+#     * Integrate any additional instructions provided by the user.
+
+# ---
+
+# ## Output Format (Plain Text Notebook Plan)
+
+# Your output must be a single, continuous text document representing the notebook plan.
+# Do NOT output JSON. Follow this strict markdown structure:
+
+# -   **First-level headings (`#`):** For main sections (e.g., `# 1. Problem Statement`).
+# -   **Second-level headings (`##`):** For subsections (e.g., `## Business Context`).
+# -   **Code Cell Details:** For every operation that requires a code cell, describe it using the format:
+#     `Code Cell X: <Detailed, prescriptive description of the operation>`
+#     -   `X` should be a sequential number within that specific subsection.
+#     -   The description must be **verbose and explicit**, detailing *what* the user should perform in that cell, *why* (linking to problem/data), and *how* (conceptually, not actual code).
+#     -   **Example:** "Code Cell 1: In this cell, we will load the dataset from '{dataset_path}' using the `pandas.read_csv()` function. This is crucial as it brings our raw data into a DataFrame for initial inspection and subsequent analysis, directly addressing the need to analyze '{dataset_type}' data for the '{problem}' problem."
+
+# ---
+
+# ## Key Constraints for Plan Generation
+
+# * **Granularity:** Expand each conceptual step from the blueprint into multiple granular code cells if needed, aiming for the 70-80 cell target.
+# * **Specificity:** Refer to specific columns and data types from the `data_dictionary` and inferred dataset characteristics.
+# * **Relevance:** Every operation must be justified by its relevance to the `problem_statement` and the `dataset`.
+# * **Scope:** **ABSOLUTELY DO NOT INTRODUCE CONCEPTS, LIBRARIES, OR TECHNIQUES THAT ARE NOT EXPLICITLY MENTIONED OR IMPLICITLY COVERED WITHIN THE `Topics and Subtopics` provided.** This is paramount.
+# * **Pedagogical Soundness:** The plan should logically progress, making sense for a learner.
+# * **No Actual Code:** Provide detailed *instructions* for the code, not the code itself.
+# * **No Commentary:** Do not include any conversational filler or extra markdown outside the specified plan structure.
+
+# Now, generate the detailed notebook plan as a plain text document.
+# """.strip()
+
+#     # 4. Invoke LLM
+#     # Assuming 'model' is your configured LLM client
+#     response = model.invoke([SystemMessage(content=prompt)])
+#     plan_text = response.content
+
+#     # 5. Update state
+#     state["plan"] = plan_text
+#     # state["_plan_approved"] = False # Reset approval status
+#     # state["plan_feedback"] = ""     # Clear feedback
+
+#     return state
+
+
+# ## **Review Plan**
+# def review_plan_node(state: InteractiveCaseStudyState) -> InteractiveCaseStudyState:
+#     """
+#     Review‑Plan Node (UI‑driven):
+#     - Expects that the front‑end will render the JSON plan for user review.
+#     - The front‑end should display two buttons: 'YES' and 'NO'.
+#     - If 'NO' is clicked, the front‑end should show a text input for feedback.
+#     - Stores:
+#         state['_plan_approved'] = True/False
+#         state['plan_feedback'] = "" or the user’s feedback.
+#     """
+#     # 1) Grab the generated plan
+#     plan = state.get("plan", {})
+
+#     # 2) In a console environment, we’d pretty‑print it; in Streamlit, you’d st.json(plan)
+#     print("\n===== PROPOSED NOTEBOOK PLAN =====")
+#     print(json.dumps(plan, indent=2))
+#     print("==================================\n")
+
+#     # 3) UI layer responsibility:
+#     #    Render two buttons: YES / NO.
+#     #    If YES: call this node again with user_action="yes"
+#     #    If NO: call this node again with user_action="no" and then collect feedback.
+
+#     # For console fallback, we still allow text input:
+#     ans = input("Do you approve this plan? (YES/NO): ").strip().lower()
+#     approved = ans in ("yes", "y")
+#     state["_plan_approved"] = approved
+
+#     if not approved:
+#         # show feedback prompt
+#         feedback = input("Please provide feedback to improve the plan:\n")
+#         state["plan_feedback"] = feedback
+#     else:
+#         # clear any old feedback
+#         state["plan_feedback"] = ""
+
+#     return state
+
+
+
+## **Enhance Plan**
+def enhance_plan_node(state):
+    """
+    Enhance‑Plan Node:
+    - Inputs:
+      • state["plan"]: the last JSON plan
+      • state["plan_feedback"]: user’s free‑text feedback on how to improve it
+      • state["domain"], state["topics_and_subtopics"], state["problem_statement"]
+      • state["dataset_type"], state["dataset_file_path"], state["data_dictionary"]
+      • state["additional_instructions"]
+    - Action: Re‑prompt the LLM, injecting the feedback to revise the plan.
+    - Output: Overwrites state["plan"] with the refined JSON,
+      clears plan_feedback, and resets state["_plan_approved"] to False.
+    """
+
+    # 1. Extract only the needed inputs
+    original_plan      = state["plan"]
+    feedback           = state.get("plan_feedback", "")
+    domain             = state["domain"]
+    topics             = state["topics_and_subtopics"]
+    problem            = state["problem_statement"]
+    dataset_type       = state["dataset_type"]
+    dataset_path       = state["dataset_file_path"]
+    data_dict          = state.get("data_dictionary", {})
+    additional_instr   = state.get("additional_instructions", "")
+
+    # 2. Build the enhancement prompt
+    prompt = f"""
+You are an expert AI notebook planner. The user has reviewed the following plan
+and given feedback on how to improve it:
+
+---
+**Original Plan** (JSON):
+{json.dumps(original_plan, indent=2)}
+
+**User Feedback**:
+{feedback}
+
+---
+**Context for Revision**:
+- Domain: {domain}
+- Topics & Subtopics: {topics}
+- Problem Statement: {problem}
+- Dataset Type: {dataset_type}
+- Dataset Location: {dataset_path}
+- Data Dictionary:
+{json.dumps(data_dict, indent=2)}
+- Additional Instructions:
+{additional_instr if additional_instr.strip() else "None"}
+
+Please revise the original plan to address the user’s feedback.
+- Keep the same JSON structure (top‑level "sections", each with "title", "type", "subsections").
+- Adjust section titles, ordering, and code_rules as needed to satisfy the feedback.
+- Do not add sections beyond the user’s specified topics.
+- Return **only** the updated plan JSON — no extra commentary.
+""".strip()
+
+    # 3. Invoke the LLM
+    response = model.invoke([SystemMessage(content=prompt)])
+    new_plan = extract_json_from_response(response.content)
+
+    # 4. Save and reset flags
+    state["plan"]           = new_plan
+    # state["_plan_approved"] = False
+    # state["plan_feedback"]  = ""
+
+    return state
+
+
+
+## **Write Code**
+def write_code_node(state: InteractiveCaseStudyState) -> InteractiveCaseStudyState:
+    """
+    Write‑Code Node:
+    - Reads state["plan"] plus user inputs (dataset_type, dataset_file_path,
+      data_dictionary, domain, additional_instructions).
+    - Prompts the LLM to generate a flattened Jupyter notebook JSON skeleton (nbformat v4)
+      where each task becomes its own cell:
+        • A Markdown cell for each section heading (# Section)
+        • A Markdown cell for each subsection heading (## Subsection)
+        • For `type=="markdown"` subsections: one Markdown cell containing the description
+        • For `type=="code"` subsections: one code cell per rule, with inline comments
+          explaining each line of code
+        • A final Markdown cell `# Insights & Recommendations`
+      All code cells must have execution_count=null and outputs=[].
+    - Stores the result in state["raw_notebook"].
+    - Resets notebook review flags for the UI.
+    """
+
+    # 1) Extract inputs
+    plan                = state["plan"]
+    dataset_type        = state["dataset_type"]
+    dataset_path        = state["dataset_file_path"]
+    data_dict           = state.get("data_dictionary", {})
+    domain              = state["domain"]
+    additional_instr    = state.get("additional_instructions", "")
+
+
+    # 2) Build the comprehensive prompt
+    prompt = f"""
+You are an expert notebook generator. Generate a Jupyter notebook JSON (nbformat v4)
+that implements the following plan exactly and by flattening each plan subsection into individual cells.
+Do NOT include any outputs—only code & markdown cells.
+
+PLAN:
+{json.dumps(plan, indent=2)}
+
+CONTEXT:
+- Domain: {domain}
+- Dataset type: {dataset_type}
+- Dataset location: {dataset_path}
+- Data Dictionary:
+{json.dumps(data_dict, indent=2)}
+- Additional Instructions:
+{additional_instr if additional_instr.strip() else "None provided"}
+
+OUTPUT SPEC:
+1. Return only a valid notebook JSON with keys: nbformat, nbformat_minor, metadata, cells.
+2. For each section in PLAN:
+   a. Add a **markdown** cell: `# <Section Title>`
+   b. For each subsection:
+      i. Add a **markdown** cell: `## <Subsection Title>`
+     ii. If subsection `type` is "markdown":
+         - Add a separate markdown cell with the `description` text.
+    iii. If subsection `type` is "code":
+         - For each rule in `code_rules`:
+             • Create one **code** cell.
+             • Write only the code implementing that rule.
+             • Ensure each line of code has an appropriate inline comment explaining its purpose.
+3. After all sections, append one **markdown** cell:
+   `# Insights & Recommendations`
+4. Each **code** cell must include:
+   - `"cell_type": "code"`, `"execution_count": null`, `"metadata": `, `"outputs": []`, `"source": […]`
+5. Each **markdown** cell must include:
+   - `"cell_type": "markdown"`, `"metadata": `, `"source": […]`
+
+Return **only** the complete notebook JSON object.
+"""
+
+    # 3) Invoke the model
+    response = model.invoke([SystemMessage(content=prompt)])
+
+    # 4) Parse and store the raw notebook skeleton
+    raw_nb = extract_json_from_response(response.content)
+    state["raw_notebook"] = raw_nb
+
+    # 5) Reset review flags so the UI will prompt the user next
+    # state["_notebook_approved"] = False
+    # state["notebook_feedback"] = ""
+
+    return state
+
+
+
+## **Review Notebook**
+# def review_notebook_node(state: InteractiveCaseStudyState) -> InteractiveCaseStudyState:
+#     """
+#     Review-Notebook Node:
+#     - Reads state["raw_notebook"] (the skeleton JSON).
+#     - Expects the UI to render that notebook for the user.
+#     - Reads two new state flags set by the UI:
+#         • state["_notebook_approved"] (bool)
+#         • state["notebook_feedback"] (str, empty if approved)
+#     - Returns the updated state for downstream branching.
+#     """
+
+#     # 1) Sanity check
+#     raw_nb = state.get("raw_notebook")
+#     if raw_nb is None:
+#         raise ValueError("No raw_notebook found in state")
+
+#     # At this point your Streamlit app should:
+#     #   • Render `raw_nb` as a notebook preview
+#     #   • Present two buttons: “Yes” and “No”
+#     #   • If “No” is clicked, show a text_area for feedback
+
+#     # 2) Read back user’s choice from the state (set by the UI)
+#     approved = state.get("_notebook_approved", False)
+#     feedback = state.get("notebook_feedback", "").strip()
+
+#     # 3) Ensure feedback is only set when rejected
+#     if approved:
+#         # clear any spurious feedback
+#         state["notebook_feedback"] = ""
+#     else:
+#         # if user hasn’t provided feedback yet, ensure it’s initialized
+#         state["notebook_feedback"] = feedback
+
+#     # 4) State flags remain as set by the UI.
+#     # Downstream graph will branch on _notebook_approved.
+
+#     return state
+
+
+
+## **Modify Notebook**
+def modify_notebook_node(state: InteractiveCaseStudyState) -> InteractiveCaseStudyState:
+    """
+    Modify-Notebook Node:
+    - Reads the existing notebook skeleton from state["raw_notebook"].
+    - Reads user feedback from state["notebook_feedback"].
+    - Uses additional context (domain, topics, problem, dataset_path, data_dictionary)
+      to re-prompt the LLM to refine the notebook skeleton.
+    - Overwrites state["raw_notebook"] with the updated JSON.
+    - Resets approval flag for the next review cycle.
+    """
+
+    # 1. Extract required fields
+    raw_nb       = state.get("raw_notebook")
+    feedback     = state.get("notebook_feedback", "").strip()
+    domain       = state.get("domain", "")
+    topics       = state.get("topics_and_subtopics", "")
+    problem      = state.get("problem_statement", "")
+    dataset_path = state.get("dataset_file_path", "")
+    data_dict    = state.get("data_dictionary", {})
+
+    if raw_nb is None:
+        raise ValueError("No raw_notebook in state to modify")
+    if not feedback:
+        raise ValueError("No notebook_feedback in state to guide modification")
+
+    # 2. Build the prompt with extra context
+    prompt = f"""
+You are an expert AI notebook refiner. The user is working in the **{domain}** domain,
+with the following context:
+
+• **Topics & Subtopics:**
+{topics}
+
+• **Problem Statement:**
+{problem}
+
+• **Dataset Location:**
+{dataset_path}
+
+• **Data Dictionary:**
+{json.dumps(data_dict, indent=2)}
+
+The user reviewed the current Jupyter notebook skeleton (nbformat v4).
+They provided the following feedback on what to change:
+
+---
+**User Feedback:**
+{feedback}
+
+---
+**Current Notebook Skeleton (JSON):**
+{json.dumps(raw_nb, indent=2)}
+
+---
+Please produce a revised notebook skeleton JSON that:
+1. Incorporates the user’s feedback precisely.
+2. Maintains valid nbformat v4 structure (keys: nbformat, nbformat_minor, metadata, cells).
+3. Preserves existing code and markdown formatting except where edits are requested.
+4. Uses domain‑appropriate libraries and patterns (per the domain context above).
+5. Outputs **only** the updated JSON object — no extra text or commentary.
+"""
+
+    # 3. Invoke the LLM
+    response = model.invoke([SystemMessage(content=prompt)])
+    updated_nb = extract_json_from_response(response.content)
+
+    # 4. Store updated skeleton & reset approval flag
+    state["raw_notebook"]       = updated_nb
+    state["_notebook_approved"] = False
+
+    return state
+
+
+
+# # **Execute Notebook**
+# def execute_notebook_node(state: InteractiveCaseStudyState) -> InteractiveCaseStudyState:
+#     """
+#     Execute-Notebook Node:
+#     - Reads state['raw_notebook'] (nbformat v4 JSON, cells with empty outputs).
+#     - Writes it to a timestamped temp .ipynb alongside the dataset.
+#     - Runs it via Papermill, capturing outputs.
+#     - On success: stores full executed notebook in state['executed_notebook'].
+#     - On failure: stores the partially executed notebook in state['executed_notebook']
+#                   and the exception text in state['execution_error'].
+#     """
+
+#     # 1. Extract inputs
+#     raw_nb       = state.get("raw_notebook", {})
+#     dataset_path = state.get("dataset_file_path", "")
+
+#     # 2. Prepare unique temp directory
+#     run_id = datetime.now().strftime("%Y%m%d_%H%M%S")
+#     tmp_dir = f"tmp_notebooks/run_{run_id}"
+#     os.makedirs(tmp_dir, exist_ok=True)
+
+#     raw_path  = os.path.join(tmp_dir, "raw_notebook.ipynb")
+#     exec_path = os.path.join(tmp_dir, "executed_notebook.ipynb")
+
+#     # 3. Write the skeleton notebook to disk
+#     nb_node = nbformat.from_dict(raw_nb)
+#     nb_node.metadata.setdefault("kernelspec", {"name": "python3"})
+#     nb_node.metadata.setdefault("language_info", {"name": "python"})
+#     with open(raw_path, "w", encoding="utf-8") as f:
+#         nbformat.write(nb_node, f)
+
+#     # 4. Stage dataset file
+#     if dataset_path and os.path.exists(dataset_path):
+#         shutil.copy(dataset_path, tmp_dir)
+
+#     # 5. Execute via Papermill
+#     try:
+#         pm.execute_notebook(
+#             input_path=raw_path,
+#             output_path=exec_path,
+#             parameters={},
+#             log_output=True,
+#             kernel_name="python3"
+#         )
+#         # 6a. Success: read full executed notebook
+#         executed_nb = nbformat.read(exec_path, as_version=4)
+#         state["executed_notebook"] = nbformat.writes(executed_nb)
+#         state.pop("execution_error", None)
+#     except Exception as e:
+#         # 6b. Failure: read whatever Papermill wrote, or fallback to skeleton
+#         if os.path.exists(exec_path):
+#             partial_nb = nbformat.read(exec_path, as_version=4)
+#         else:
+#             partial_nb = nbformat.read(raw_path, as_version=4)
+
+#         state["executed_notebook"] = nbformat.writes(partial_nb)
+#         state["execution_error"]    = str(e)
+
+#     return state
+
+
+import os
+import sys
+import subprocess
+import shutil
+import json
+import re
+import nbformat
+from pathlib import Path
+from datetime import datetime
+import papermill as pm
+
+def execute_notebook_node(state: InteractiveCaseStudyState) -> InteractiveCaseStudyState:
+    """
+    Execute-Notebook Node with venv isolation and proper pip-install handling:
+    - Reads state['raw_notebook'] (nbformat v4 JSON).
+    - Writes it to a timestamped temp folder as raw_notebook.ipynb.
+    - Creates a Python venv in that folder.
+    - Detects any '!pip install' lines: extracts package names and installs them in venv.
+    - Comments out the original '!pip install' lines in the notebook to avoid re-running them.
+    - Registers an ipykernel for the venv (if possible).
+    - Runs the notebook via Papermill under that venv/kernel, capturing outputs.
+    - On success: stores executed notebook JSON in state["executed_notebook"], clears state["execution_error"].
+    - On failure: injects an error markdown cell at the top, stores partially executed notebook JSON in state["executed_notebook"], and error text in state["execution_error"].
+    """
+    raw_nb = state.get("raw_notebook", {})
+    dataset_path = state.get("dataset_file_path", "")
+
+    # 1. Prepare unique temp directory
+    run_id = datetime.now().strftime("%Y%m%d_%H%M%S")
+    tmp_dir = Path(f"tmp_notebooks/run_{run_id}")
+    tmp_dir.mkdir(parents=True, exist_ok=True)
+
+    raw_path = tmp_dir / "raw_notebook.ipynb"
+    exec_path = tmp_dir / "executed_notebook.ipynb"
+
+    # 2. Write the skeleton notebook to disk
+    nb_node = nbformat.from_dict(raw_nb)
+    nb_node.metadata.setdefault("kernelspec", {"name": "python3"})
+    nb_node.metadata.setdefault("language_info", {"name": "python"})
+    with raw_path.open("w", encoding="utf-8") as f:
+        nbformat.write(nb_node, f)
+
+    # 3. Stage dataset file if exists
+    if dataset_path and os.path.exists(dataset_path):
+        try:
+            shutil.copy(dataset_path, tmp_dir)
+        except Exception:
+            pass  # ignore if copy fails
+
+    # 4. Helpers to detect and comment out '!pip install' lines
+    def extract_pip_packages(ipynb_path: Path) -> list:
+        """Return list of package strings from '!pip install ...' lines."""
+        pkgs = []
+        try:
+            notebook = json.loads(ipynb_path.read_text(encoding="utf-8"))
+        except Exception:
+            return pkgs
+        pattern = r'!pip\s+install\s+(.+)'
+        for cell in notebook.get('cells', []):
+            if cell.get('cell_type') == 'code':
+                source = ''.join(cell.get('source', []))
+                for line in source.splitlines():
+                    m = re.match(pattern, line.strip())
+                    if m:
+                        rest = m.group(1).strip()
+                        # Remove inline comments after '#'
+                        rest = rest.split('#', 1)[0].strip()
+                        # Split by whitespace to get package tokens
+                        parts = rest.split()
+                        for p in parts:
+                            if p:
+                                pkgs.append(p)
+        return pkgs
+
+    def comment_out_pip_lines(ipynb_path: Path):
+        """
+        Modify the notebook in-place: prefix any '!pip install...' lines with '# ',
+        preserving indentation, so they won't run.
+        """
+        try:
+            nb = nbformat.read(str(ipynb_path), as_version=4)
+        except Exception:
+            return
+        modified = False
+        for cell in nb.cells:
+            if cell.cell_type == 'code':
+                lines = cell.source.splitlines()
+                new_lines = []
+                for line in lines:
+                    # Detect leading whitespace
+                    leading_ws = line[:len(line) - len(line.lstrip())]
+                    stripped = line.lstrip()
+                    if stripped.startswith("!pip install"):
+                        # Comment out, preserving indentation
+                        new_line = leading_ws + "# " + stripped
+                        new_lines.append(new_line)
+                        modified = True
+                    else:
+                        new_lines.append(line)
+                cell.source = "\n".join(new_lines)
+        if modified:
+            try:
+                nbformat.write(nb, str(ipynb_path))
+            except Exception:
+                pass
+
+    try:
+        # 5. Create and prepare virtual environment
+        env_path = tmp_dir / "venv"
+        if not env_path.exists():
+            subprocess.run([sys.executable, "-m", "venv", str(env_path)], check=True)
+
+        # Determine python executable in venv
+        if os.name == "nt":
+            python_exec = env_path / "Scripts" / "python.exe"
+        else:
+            python_exec = env_path / "bin" / "python"
+
+        # 6. Bootstrap pip in venv (if needed)
+        try:
+            subprocess.run([str(python_exec), "-m", "ensurepip", "--upgrade"], check=True)
+        except Exception:
+            pass
+        try:
+            subprocess.run([str(python_exec), "-m", "pip", "install", "--upgrade", "pip"], check=True)
+        except Exception:
+            pass
+
+        # 7. Detect packages from '!pip install' lines and install them in venv
+        packages = extract_pip_packages(raw_path)
+        if packages:
+            install_cmd = [str(python_exec), "-m", "pip", "install"] + packages
+            try:
+                subprocess.run(install_cmd, check=True, capture_output=True, text=True)
+            except subprocess.CalledProcessError as e:
+                err = e.stderr or e.stdout or str(e)
+                # Record installation error in state; execution may still proceed and fail later
+                prev = state.get("execution_error", "")
+                state["execution_error"] = (prev + "\n" if prev else "") + f"Failed to install packages {packages}: {err}"
+            # After installing, comment out original pip lines so notebook won't try again
+            comment_out_pip_lines(raw_path)
+
+        # 8. Install papermill, ipykernel, nbformat in venv so we can run papermill under venv
+        try:
+            subprocess.run(
+                [str(python_exec), "-m", "pip", "install", "papermill", "ipykernel", "nbformat"],
+                check=True, capture_output=True, text=True
+            )
+        except Exception as e:
+            err = getattr(e, 'stderr', None) or str(e)
+            prev = state.get("execution_error", "")
+            state["execution_error"] = (prev + "\n" if prev else "") + f"Failed to install papermill/ipykernel: {err}"
+
+        # 9. Register ipykernel for this venv (optional; if fails, Papermill may use default kernel)
+        kernel_name = f"venv_{run_id}"
+        try:
+            subprocess.run(
+                [
+                    str(python_exec), "-m", "ipykernel", "install",
+                    "--user",
+                    "--name", kernel_name,
+                    "--display-name", f"Python ({kernel_name})"
+                ],
+                check=True, capture_output=True, text=True
+            )
+        except Exception as e:
+            err = getattr(e, 'stderr', None) or str(e)
+            prev = state.get("execution_error", "")
+            state["execution_error"] = (prev + "\n" if prev else "") + f"Failed to register ipykernel: {err}"
+
+        # 10. Execute via Papermill under venv
+        input_nb = raw_path.resolve()
+        output_nb = exec_path.resolve()
+        cmd = [str(python_exec), "-m", "papermill", str(input_nb), str(output_nb), "-k", kernel_name]
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                cwd=str(tmp_dir),
+                timeout=60 * 30  # adjust as needed
+            )
+            stderr = result.stderr or ""
+            returncode = result.returncode
+
+            # 11. Read output notebook if created
+            if output_nb.exists():
+                executed_nb = nbformat.read(str(output_nb), as_version=4)
+            else:
+                # Fallback to raw or partially executed
+                executed_nb = nbformat.read(str(raw_path), as_version=4)
+
+            # 12. Handle return code: if non-zero, inject error cell
+            if returncode != 0:
+                error_msg = f"Papermill exited with code {returncode}.\nStderr:\n{stderr}"
+                err_cell = nbformat.v4.new_markdown_cell(f"**Execution Error:**\n```\n{error_msg}\n```")
+                executed_nb.cells.insert(0, err_cell)
+                prev = state.get("execution_error", "")
+                state["execution_error"] = (prev + "\n" if prev else "") + error_msg
+            else:
+                # Success: clear any previous execution_error
+                state.pop("execution_error", None)
+
+            # 13. Save executed notebook JSON into state
+            state["executed_notebook"] = nbformat.writes(executed_nb)
+
+        except subprocess.TimeoutExpired as te:
+            # Timeout: capture partial output and inject timeout error
+            if exec_path.exists():
+                partial_nb = nbformat.read(str(exec_path), as_version=4)
+            else:
+                partial_nb = nbformat.read(str(raw_path), as_version=4)
+            timeout_msg = f"Timeout: notebook execution exceeded limit ({te})."
+            err_cell = nbformat.v4.new_markdown_cell(f"**Execution Error:**\n```\n{timeout_msg}\n```")
+            partial_nb.cells.insert(0, err_cell)
+            state["executed_notebook"] = nbformat.writes(partial_nb)
+            state["execution_error"] = timeout_msg
+
+        except Exception as e:
+            # General execution error: inject error cell
+            if exec_path.exists():
+                partial_nb = nbformat.read(str(exec_path), as_version=4)
+            else:
+                partial_nb = nbformat.read(str(raw_path), as_version=4)
+            err = str(e)
+            err_cell = nbformat.v4.new_markdown_cell(f"**Execution Error:**\n```\n{err}\n```")
+            partial_nb.cells.insert(0, err_cell)
+            state["executed_notebook"] = nbformat.writes(partial_nb)
+            state["execution_error"] = err
+
+    except Exception as e:
+        # Any errors in venv setup or earlier steps: inject error at top of raw notebook
+        raw_nb_node = nbformat.from_dict(raw_nb)
+        err = str(e)
+        err_cell = nbformat.v4.new_markdown_cell(f"**Execution Setup Error:**\n```\n{err}\n```")
+        raw_nb_node.cells.insert(0, err_cell)
+        state["executed_notebook"] = nbformat.writes(raw_nb_node)
+        state["execution_error"] = f"Setup failed: {err}"
+
+    return state
+
+
+
+## **Review Execution**
+# def review_execution_node(state: InteractiveCaseStudyState) -> InteractiveCaseStudyState:
+#     """
+#     Review-Execution Node:
+#     - Reads state['executed_notebook'] (the .ipynb JSON with outputs).
+#     - Reads state.get('execution_error') to know if something failed.
+#     - Updates:
+#         state['_execution_approved']: bool
+#         state['execution_feedback']: str (if any)
+#     """
+
+#     # 1. Extract executed notebook and error flag
+#     executed_nb_json = state.get("executed_notebook", "")
+#     error_msg        = state.get("execution_error")
+
+#     # 2. Validate JSON
+#     try:
+#         nb = json.loads(executed_nb_json)
+#     except Exception:
+#         # Malformed notebook JSON → automatic reject
+#         state["_execution_approved"] = False
+#         state["execution_feedback"]  = "Executed notebook JSON could not be parsed."
+#         return state
+
+#     # 3. Decide success vs failure
+#     if error_msg:
+#         # FAILURE scenario
+#         # In Streamlit you’d render `nb` plus highlight error_msg prominently.
+#         print("😢 Sadly, an error occurred while executing the notebook:")
+#         print(f">>> {error_msg}\n")
+#         # Prompt user: feedback or auto‑handle
+#         ans = input(
+#             "Do you have feedback on this error, or should I handle it myself? (feedback/auto): "
+#         ).strip().lower()
+#         if ans == "auto":
+#             # Let the system attempt auto‑correction
+#             state["_execution_approved"] = False
+#             state["execution_feedback"]  = ""  # no user feedback
+#         else:
+#             # Collect user feedback
+#             feedback = input("Please describe how I should fix this:\n").strip()
+#             state["_execution_approved"] = False
+#             state["execution_feedback"]  = feedback or "No feedback provided."
+#     else:
+#         # SUCCESS scenario
+#         # Streamlit: celebrate with "Notebook executed successfully end‑to‑end!"
+#         print("🎉 Notebook executed successfully end‑to‑end!")
+#         # Ask for final proceed approval
+#         ans = input("Shall I proceed to write the insights? (YES/NO): ").strip().lower()
+#         state["_execution_approved"] = (ans in ("yes", "y"))
+#         if not state["_execution_approved"]:
+#             # If user declines, optionally capture why
+#             feedback = input("Please share any concerns before writing insights:\n").strip()
+#             state["execution_feedback"] = feedback or ""
+
+#     return state
+
+
+
+## **Correct Notebook**
+def correct_notebook_node(state: InteractiveCaseStudyState) -> InteractiveCaseStudyState:
+    """
+    Correct-Notebook Node:
+    - Inputs:
+        • state['raw_notebook']: the notebook skeleton JSON (cells only, no outputs)
+        • state['execution_error']: the error string produced by Papermill
+        • state['execution_feedback']: optional user feedback string from review_execution_node
+    - Action:
+        • Prompt an LLM to minimally patch the notebook JSON to fix errors.
+    - Output:
+        • Overwrites state['raw_notebook'] with corrected notebook JSON.
+        • Resets state['_execution_approved'] to False to trigger re-execution.
+    """
+
+    # 1. Extract from state
+    raw_nb = state.get("raw_notebook", {})
+    exec_error = state.get("execution_error", "").strip()
+    feedback = state.get("execution_feedback", "").strip()
+
+    if not raw_nb:
+        raise ValueError("Missing required key: 'raw_notebook'")
+
+    # 2. Build the LLM prompt
+    prompt = f"""
+You are a Python notebook repair assistant.
+
+Below is the original notebook code (nbformat v4 JSON):
+{json.dumps(raw_nb, indent=2)}
+
+The following error occurred during execution:
+{exec_error or '(no error provided)'}
+
+{('In addition, the user provided the following correction feedback:' if feedback else 'The user did not provide any manual feedback.')}
+
+Please return a corrected version of the notebook (still in nbformat v4 JSON).
+Make only the **minimal** necessary changes to fix the above error and satisfy the feedback if present.
+Preserve all other cell content and metadata exactly as they are.
+
+Return only the JSON. Do not include any extra explanation or comments.
+"""
+
+    # 3. Send to LLM
+    response = model.invoke([SystemMessage(content=prompt)])
+    corrected_nb = extract_json_from_response(response.content)
+
+    # 4. Save updated notebook back into state
+    state["raw_notebook"] = corrected_nb
+    state["_execution_approved"] = False  # force re-execution after correction
+
+    return state
+
+
+
+## **Write Insights**
+# ─────────────── write_insights_node ──────────────────
+def write_insights_node(state: InteractiveCaseStudyState) -> InteractiveCaseStudyState:
+    """
+    Write-Insights Node:
+    - Inputs: state['executed_notebook'] (JSON str), state['problem_statement'], state['dataset_path']
+    - Stages dataset for any REPL plotting.
+    - For each code cell with outputs:
+        • If plot: re‑execute all code so far via PythonREPL agent → insight.
+        • Else: call LLM with problem, prior observations, code, and raw output.
+    - Inserts a markdown “Observation” after each code cell.
+    - Appends a final "# Insights and Conclusion" cell.
+    - Sets state['final_notebook'] to the updated notebook JSON (as Python dict).
+    """
+    # 1) Load executed notebook JSON
+    executed_nb_str = state.get("executed_notebook", "")
+    executed_nb = json.loads(executed_nb_str)
+
+    problem_stmt = state.get("problem_statement", "")
+    dataset_path = state.get("dataset_file_path", "")
+    cells = executed_nb.get("cells", [])
+
+    # 2) Prepare for plot‑REPL (commented out if not needed here)
+    work_dir = "tmp_repl"
+    # stage_dependencies(dataset_path, work_dir)
+    # plot_executor = build_plot_insight_agent_executor(model)
+
+    updated_cells = []
+    context_history = []   # list of past observation strings
+    code_history = []      # list of past code sources
+
+    for cell in cells:
+        updated_cells.append(cell)
+
+        if cell.get("cell_type") == "code" and cell.get("outputs"):
+            code_src = "".join(cell.get("source", []))
+            code_history.append(code_src)
+
+            # aggregate textual outputs
+            raw_out = ""
+            for out in cell["outputs"]:
+                if out.get("output_type") == "stream":
+                    raw_out += "".join(out.get("text", []))
+                elif out.get("output_type") == "execute_result":
+                    raw_out += "".join(out.get("data", {}).get("text/plain", []))
+
+            if is_plot_code(code_src):
+                # Commented out observation writing for plot cells
+                # combined = "\n".join(code_history)
+                # cwd = os.getcwd()
+                # os.chdir(work_dir)
+                # tool_call = plot_executor.invoke({"input": combined})
+                # obs = tool_call.get("output", "").strip()
+                # os.chdir(cwd)
+                obs = ""  # No observation added for plot cells
+            else:
+                prior = "\n".join(f"- {o}" for o in context_history)
+                prompt = f"""
+You are an AI assistant writing a single one line bullet‑point observation (no code).
+
+Problem Statement:
+{problem_stmt}
+
+Prior Observations:
+{prior}
+
+Current Code:
+{code_src}
+
+Current Output:
+{raw_out}
+
+Write one concise, beginner‑friendly bullet‑point that interprets this output
+in context of the problem.
+"""
+                resp = model.invoke([SystemMessage(content=prompt)])
+                obs = resp.content.strip()
+
+            if obs:
+                observation_cell = {
+                    "cell_type": "markdown",
+                    "metadata": {},
+                    "source": [f"**Observation:** {obs}\n"]
+                }
+                updated_cells.append(observation_cell)
+                context_history.append(obs)
+
+    # Final summary section
+    all_obs = "\n".join(f"- {o}" for o in context_history)
+    summary_prompt = f"""
+You are an AI assistant composing the final **Insights and Conclusion** section.
+
+Problem Statement:
+{problem_stmt}
+
+Observations:
+{all_obs}
+
+Provide a short set of business‑focused recommendations and a conclusion.
+"""
+    summary_resp = model.invoke([SystemMessage(content=summary_prompt)])
+    summary_text = summary_resp.content.strip()
+    summary_cell = {
+        "cell_type": "markdown",
+        "metadata": {},
+        "source": [f"# Insights and Conclusion\n\n{summary_text}\n"]
+    }
+    updated_cells.append(summary_cell)
+
+    executed_nb["cells"] = updated_cells
+    state["final_notebook"] = executed_nb
+    return state
+
+# ## **Final Workflow**
+
+# from langgraph.graph import StateGraph, START, END
+# from IPython.display import Image, display
+# from langgraph.checkpoint.memory import InMemorySaver
+
+
+# def create_interactive_workflow() -> StateGraph:
+#     graph = StateGraph(InteractiveCaseStudyState)
+
+#     # Core nodes
+#     graph.add_node("think_sections",  think_sections_node)
+#     graph.add_node("review_plan",     review_plan_node)
+#     graph.add_node("enhance_plan",    enhance_plan_node)
+#     graph.add_node("write_code",      write_code_node)
+#     graph.add_node("review_notebook", review_notebook_node)
+#     graph.add_node("modify_notebook", modify_notebook_node)
+#     graph.add_node("execute_notebook",execute_notebook_node)
+#     graph.add_node("review_execution",review_execution_node)
+#     graph.add_node("correct_notebook",correct_notebook_node)
+#     graph.add_node("write_insights",  write_insights_node)
+
+#     # START → Think‑Sections → Review Plan
+#     graph.add_edge(START, "think_sections")
+#     graph.add_edge("think_sections", "review_plan")
+
+#     graph.add_conditional_edges(
+#     "review_plan",
+#     lambda s: "true" if s.get("_plan_approved", False) else "false",
+#     {"true": "write_code", "false": "enhance_plan"}
+#     )
+#     graph.add_edge("enhance_plan", "review_plan")
+
+#     # Write‑Code → Review Notebook
+#     graph.add_edge("write_code", "review_notebook")
+#     graph.add_conditional_edges(
+#         "review_notebook",
+#         lambda s: "true" if s.get("_notebook_approved", False) else "false",
+#         {"true": "execute_notebook", "false": "modify_notebook"}
+#     )
+#     graph.add_edge("modify_notebook", "review_notebook")
+
+#     # Execute Notebook → Review Execution
+#     graph.add_edge("execute_notebook", "review_execution")
+#     graph.add_conditional_edges(
+#         "review_execution",
+#         lambda s: "true" if s.get("_execution_approved", False) else "false",
+#         {"true": "write_insights", "false": "correct_notebook"}
+#     )
+#     graph.add_edge("correct_notebook", "execute_notebook")
+
+#     # Final Insights → END
+#     graph.add_edge("write_insights", END)
+
+#     return graph
+
+# from langgraph.checkpoint.memory import InMemorySaver
+# checkpointer = InMemorySaver()
+
+# # ─────────── Compile & Visualize ───────────
+# case_study_workflow = create_interactive_workflow().compile(checkpointer=checkpointer)
+# # Show the Mermaid diagram of the workflow
+# display(Image(case_study_workflow.get_graph().draw_mermaid_png()))
+
+# thread = {"configurable": {"thread_id": "1"}}
+
+# case_study_workflow.get_state(thread).next
+
+# case_study_workflow.get_state({"configurable": {"thread_id": "1"}}).next
+
+

src/workflow_utils.py

@@ -0,0 +1,614 @@
+# Standard Libraries
+import os
+import json
+import re
+import shutil
+
+from langchain.schema import SystemMessage
+from langchain.tools import Tool
+from langchain.agents import AgentExecutor, create_tool_calling_agent
+from langchain_experimental.utilities import PythonREPL
+
+from langchain.prompts import (
+    ChatPromptTemplate,
+    SystemMessagePromptTemplate,
+    HumanMessagePromptTemplate,
+    MessagesPlaceholder
+)
+
+## Utility Functions
+
+def extract_json_from_response(text):  # Extract and parse JSON content from a text response
+    """
+    Extract JSON content from 'text' using regex and parse it.
+    Returns the parsed JSON as a dictionary if successful, otherwise prints an error and returns None.
+    """
+    match = re.search(r"\{.*\}", text, re.DOTALL)  # Use DOTALL to match multi-line JSON blocks
+    if match:
+        try:
+            return json.loads(match.group())  # Parse and return the JSON object
+        except json.JSONDecodeError:
+            print("Error: Extracted JSON is invalid.")  # Log invalid JSON error
+            return None
+    else:
+        print("Error: No JSON found in response.")  # Log if no JSON pattern is found
+        return None
+
+
+# def generate_notebook_plan_text(
+#     plan: Dict[str, Any],
+#     output_txt_file: str = "notebook_plan.txt"
+# ) -> str:
+#     """
+#     Converts a JSON notebook plan into a human-readable text document using an LLM.
+
+#     Args:
+#         plan (Dict[str, Any]): The JSON notebook plan.
+#         output_txt_file (str): The filename to save the generated text document.
+
+#     Returns:
+#         str: The generated text content.
+#     """
+
+#     # Define the system prompt to guide the LLM's behavior
+#     system_prompt = """
+# You are an expert AI curriculum designer. Given a structured JSON notebook plan, your task is to convert it into a clear, readable text document. The document should outline the notebook's structure, including sections and subsections, specifying whether each is a markdown or code block. For code blocks, include the 'code_rules' provided. For markdown blocks, include the 'description'. Format the output with appropriate headings and bullet points for clarity.
+# """
+
+#     # Convert the JSON plan to a formatted string
+#     plan_str = json.dumps(plan, indent=2)
+
+#     # Create the messages for the LLM
+#     messages = [
+#         SystemMessage(content=system_prompt),
+#         HumanMessage(content=f"Here is the JSON notebook plan:\n\n{plan_str}")
+#     ]
+
+#     # Invoke the LLM with the messages
+#     response = model.invoke(messages)
+
+#     # Extract the generated text content
+#     text_content = response.content
+
+#     # Save the generated text to a file
+#     with open(output_txt_file, "w", encoding="utf-8") as f:
+#         f.write(text_content)
+
+#     return text_content
+
+
+
+def get_notebook_blueprint(domain: str) -> str:
+    """
+    Returns a detailed notebook blueprint string based on the specified domain.
+    """
+
+    domain_blueprints = {
+        "eda": """
+1. **Problem Statement**
+   - Business Context (markdown)
+   - Objective of the analysis (markdown)
+   - Dataset description and data dictionary overview (markdown)
+
+2. **Data Overview**
+   - Load dataset and show preview (code)
+   - Report rows, columns, and data types (code)
+   - Check for missing values and duplicates (code)
+
+3. **Univariate Analysis**
+   - Compute and plot summary statistics (histograms, box plots) (code)
+
+4. **Bivariate Analysis**
+   - Compute correlations and display heatmap (code)
+   - Scatter plots for key variable pairs (code)
+
+5. **Multivariate Analysis**
+   - Pair plots or 3D scatter plots for multi-feature interactions (code)
+
+6. **Feature Engineering & Transformation**
+   - Create new features or bucket continuous variables (code)
+   - Handle outliers or perform scaling/normalization (code)
+
+7. **Insights & Recommendations**
+   - Bullet-point summary of key findings (markdown)
+   - Business or analytical recommendations (markdown)
+""",
+        "machine_learning": """
+1. **Problem Statement**
+   - Business Context (markdown)
+   - Problem Objective (markdown)
+   - Dataset Summary (markdown)
+   - Data Dictionary (markdown)
+
+2. **Setup: Imports & Configuration**
+   - Install and import required libraries (code)
+   - Define global paths or settings (code)
+
+3. **Data Overview**
+   - Load data and show preview (code)
+   - Check for missing values and duplicates (code)
+   - Summary statistics and data types (code)
+
+4. **Exploratory Data Analysis (EDA)**
+   - Univariate: distributions, histograms (code)
+   - Bivariate/Multivariate: pair plots, heatmaps, correlation (code)
+
+5. **Modeling**
+   - Data split (train-test)
+   - Model training using selected algorithm(s)
+   - Evaluation using sutable metrics (accuracy, recall, precision, F1-score for classification and RMSE, MAE for regression)
+
+6. **Insights & Recommendations**
+   - Summary of findings
+   - Performance interpretation
+   - Recommendations for business or deployment
+""",
+
+        "deep_learning": """
+1. **Problem Statement**
+   - Business Context (markdown)
+   - Objective of using Deep Learning (markdown)
+   - Dataset Summary (markdown)
+   - Data Dictionary (markdown)
+
+2. **Setup: Imports & Configuration**
+   - Install and import required libraries (TensorFlow, Keras, PyTorch, etc.) (code)
+   - Set global seeds and GPU configuration (code)
+
+3. **Data Overview**
+   - Load data and preview (code)
+   - Check for class imbalance and missing values (code)
+   - Summary statistics and label distribution (code)
+
+4. **Preprocessing**
+   - Data cleaning and normalization/scaling (code)
+   - Label encoding or one-hot encoding (code)
+   - Reshaping input for neural networks (code)
+   - Train-validation-test split (code)
+
+5. **Model Architecture**
+   - Define sequential or functional model (code)
+   - Configure layers (Dense, Conv2D, LSTM, etc.) and activations (code)
+   - Display model summary (code)
+
+6. **Model Compilation & Training**
+   - Compile model with optimizer, loss, and metrics (code)
+   - Fit model with early stopping or callbacks (code)
+   - Plot training and validation loss/accuracy (code)
+
+7. **Evaluation**
+   - Evaluate on test set (code)
+   - Confusion matrix, classification report or regression metrics (code)
+   - Visualize predictions (code)
+
+8. **Insights & Recommendations**
+   - Interpretation of model performance (markdown)
+   - Overfitting/underfitting analysis (markdown)
+   - Suggestions for tuning or deployment (markdown)
+""",
+
+        "computer_vision": """
+
+**NOTE:** This is your high-level reference. You must **ONLY THINK, ANALYZE, AND DECIDE** based on the specific problem statement, dataset information (including its type – tabular, image, text, or mixed), and the user's specific weekly subtopics. Do NOT go beyond the weekly topics. Every decision and every operation performed must be carefully considered and justified in the context of the specific Computer Vision problem given.
+
+## 1. Problem Statement (First-level heading markdown)
+## Business Context (Second-level heading markdown)
+* Write the user’s Business context as is in a markdown cell.
+
+## Objective (Second-level heading markdown)
+* Write the user's objective as is in a markdown cell.
+
+## Data Dictionary (Second-level heading markdown)
+* Write the user's data dictionary as is in a markdown cell.
+* Enhance column/label descriptions with necessary details (e.g., class meanings, annotation formats).
+
+*(This section will have 6 markdown cells in total).*
+
+---
+
+# 2. Install and import the necessary libraries (First-level heading markdown)
+## Install Dependencies (Second-level heading markdown)
+* **Think and decide** on necessary libraries with specific versions (e.g., `tensorflow`/`torch`, `keras`, `opencv-python` (cv2), `scikit-image`, `Pillow`, `tqdm`, `albumentations` for augmentation).
+* **Code Cell:** Install all dependencies in one cell.
+
+## Import Libraries (Second-level heading markdown)
+* Import all required libraries.
+* Group imports with clear comments (e.g., "# DL Frameworks", "# Image Processing", "# Visualization").
+* Add comments for less common libraries.
+* **Code Cell:** Import all libraries in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 3. Loading the dataset (First-level heading markdown)
+* CV datasets consist of image files (e.g., `.jpg`, `.png`), often with accompanying annotation files (e.g., `.json`, `.xml`, mask images), directory or Zip file of directories structure defining classes.
+* **Think and decide** the best method to load your specific image data and annotations.
+
+## Read Data from Path/Source (Second-level heading markdown)
+* Read image data and (if applicable) annotations from the provided path/structure.
+* **Examples of operations:** `tf.keras.utils.image_dataset_from_directory` or `torchvision.datasets.ImageFolder` (classification), custom loops for reading image files and parsing annotation files (detection/segmentation).
+* **Code Cell:** Load data in one cell.
+
+## Copy Data to New Variable and Preview (Second-level heading markdown)
+* Copy data to a new variable (if applicable, e.g., list of image paths, dataframe of annotations).
+* Display a preview to confirm data integrity.
+* **Examples of operations:** Displaying a few sample images, showing image paths and corresponding labels/annotations.
+* **Code Cell:** Copy and preview data in one cell.
+
+*(This section will have 2 markdown and 2 code cells).*
+
+---
+
+# 4. Data Overview and Preliminary Checks (First-level heading markdown)
+* Analyze your image dataset: total images, typical dimensions, channels, file formats, class distribution, potential corruption, annotation format.
+* **Think, reason, and decide** on preliminary operations to show data sanity and structure.
+
+## Display Basic Dataset Information (Second-level heading markdown)
+* Display initial information about the dataset.
+* **Examples of operations:** Total number of images, number of classes (if classification), paths to a few sample images, basic info on image file types.
+* **Code Cell:** Display info in one cell.
+
+## Checking Image Characteristics (Second-level heading markdown)
+* **Think and decide** what basic image characteristics need to be checked based on your problem.
+* **Examples of checks:** Distribution of image dimensions (width/height/aspect ratio), channel distribution (RGB/Grayscale), check for corrupted/unreadable image files, summary statistics of pixel values (e.g., mean/std across channels).
+* **Code Cell(s):** Each distinct check in its own cell.
+
+## Preliminary Missing/Corrupt Samples and Duplicates (Second-level heading markdown)
+* **Think and decide** how to check for and handle missing/corrupted image files or duplicate images/annotations.
+* **Examples of checks:** Iterating through file paths to catch read errors, hashing images to find duplicates, checking for empty/malformed annotation files.
+* **Code Cell(s):** Display checks in one cell; handle issues in separate cells (if straightforward).
+
+* Identify any data quality issues (e.g., inconsistent image sizes, incorrect labels/annotations, irrelevant images).
+* **Think, analyze, and decide** which initial cleaning/conversion operations are most suitable.
+* Perform all chosen operations in separate cells. Ensure the dataset is preliminarily ready.
+* **Code Cell(s):** Perform initial quality checks and cleanups in separate cells.
+
+*(This section will have 5-7 markdown cells and 6-10 code cells. Total: 11-17 cells).*
+* **NOTE:** Focus on image-specific insights relevant to preprocessing and model building.
+
+---
+
+# 5. Exploratory Data Analysis (EDA) (First-level heading markdown)
+* **NOTE:** Do not introduce analysis beyond weekly topics.
+* Focus on key visualizations to understand image properties, label distributions, and (if applicable) annotation patterns. Aim for 5-8 plots/visualizations.
+
+## Univariate Analysis (Second-level heading markdown)
+* **Think and decide** on up to 4 plots/analyses revealing individual image or label characteristics.
+* **Examples of operations:** Bar chart of class distribution, histograms of image dimensions (width, height), pixel intensity histograms for sample images (e.g., mean pixel value distribution).
+* Justify why each visualization helps answer business questions or inform model design.
+* **Code Cell(s):** Each plot/analysis in its own cell.
+
+## Bivariate/Multivariate Analysis (Second-level heading markdown)
+* **Think and decide** on up to 4 visualizations showing relationships or patterns across multiple dimensions.
+* **Examples of operations:** Displaying sample images from different classes, visualizing bounding boxes/masks on images (for detection/segmentation), average image per class, relationships between image properties and labels.
+* Justify why each visualization helps uncover deeper insights relevant to the CV task.
+* **Code Cell(s):** Each plot/analysis in its own cell.
+
+*(This section will have 2 markdown cells and 8 code cells. Total: 10 cells).*
+
+---
+
+# 6. Data Preprocessing (First-level heading markdown)
+* Transform raw image data into a format suitable for Deep Learning models.
+* **Think, analyze, and decide** on preprocessing steps based on image data characteristics, CV task, and chosen model requirements.
+* Each distinct operation must be in its own separate code cell.
+
+## Image-Specific Transformations (Second-level heading markdown)
+* **Think and decide** on necessary transformations for your specific image data.
+* **Examples of operations:**
+    * **Resizing/Rescaling:** Standardizing image dimensions to match model input.
+    * **Pixel Normalization:** Scaling pixel values (e.g., to [0, 1] or [-1, 1]).
+    * **Channel Manipulation:** Converting to grayscale, ensuring consistent channel count (e.g., 3 channels for RGB).
+    * **Data Augmentation:** Applying transformations like rotation, flipping, zooming, brightness/contrast adjustments to increase dataset size and improve model generalization (crucial for DL in CV).
+* **Code Cell(s):** Implement these image preprocessing steps.
+
+## Annotation Preprocessing (if applicable) (Second-level heading markdown)
+* For object detection or segmentation, **think and decide** on necessary annotation processing.
+* **Examples of operations:**
+    * Parsing raw annotations (XML, JSON) into structured formats.
+    * Resizing/scaling bounding box coordinates or mask dimensions to match resized images.
+    * Converting mask images to appropriate tensor formats.
+    * Handling class IDs and one-hot encoding if needed.
+* **Code Cell(s):** Implement annotation preprocessing steps.
+
+## Data Loading Pipelines (Second-level heading markdown)
+* **Think and decide** on the most efficient way to create data pipelines for loading and batching your preprocessed image data for Deep Learning frameworks.
+* **Key Considerations:** Batching, shuffling, caching, prefetching, parallel processing to optimize training speed.
+* **Code Cell(s):** Implement your data loading pipeline (e.g., `tf.data.Dataset`, `torch.utils.data.Dataset` and `DataLoader`).
+
+*(This section will have 3-5 markdown cells and 10-15 code cells. Total: 15-20 cells).*
+* **NOTE:** Justify each preprocessing step by explaining how it prepares the data for specific DL architectures and aligns with the problem objective.
+
+---
+
+# 7. Model Building (First-level heading markdown)
+* Define and train your Computer Vision models.
+* **Think, analyze, and decide** on the most appropriate Deep Learning architecture(s) based on the CV problem, data, and objective.
+* Each distinct operation in its own cell.
+
+## Train-Validation-Test Split (Second-level heading markdown)
+* **Think and decide** on the most appropriate strategy for splitting your processed dataset into training, validation, and testing sets.
+* **Key Considerations:** Choose appropriate split ratios, use `random_state` for reproducibility, decide if stratification is necessary (especially for imbalanced classification tasks).
+* **Code Cell:** Implement the data split.
+
+## Model Architecture Definition (Second-level heading markdown)
+* **Think and decide** on the most suitable Deep Learning architecture for your CV task.
+* **Examples of architectures:**
+    * **Image Classification:** Convolutional Neural Networks (CNNs), pre-trained models from Keras Applications (e.g., ResNet, VGG, EfficientNet) or PyTorch Models.
+    * **Object Detection:** Architectures like YOLO, Faster R-CNN (often involve pre-trained backbones).
+    * **Image Segmentation:** U-Net, FCN (Fully Convolutional Networks).
+* **Considerations:** Number of layers, activation functions, regularization (Dropout, Batch Normalization), output layer matching the problem type.
+* **Code Cell:** Define your CV model architecture.
+
+## Compile Model (Second-level heading markdown)
+* **Think and decide** on the appropriate loss function and optimizer for your specific CV problem.
+* **Loss Functions:** `CategoricalCrossentropy`, `BinaryCrossentropy` (classification); `MeanSquaredError` (regression); `Dice Loss`, `IoU Loss` (segmentation); specialized losses for object detection.
+* **Optimizers:** `Adam`, `SGD` with momentum, `RMSprop`.
+* **Code Cell:** Compile the defined Deep Learning model.
+
+## Model Training (Second-level heading markdown)
+* Train your compiled Deep Learning model on the training data, using the validation set for monitoring performance.
+* **Key Training Parameters:** Epochs, Batch Size, Callbacks (e.g., `EarlyStopping`, `ModelCheckpoint`, `ReduceLROnPlateau`).
+* **Code Cell:** Write the code to train your Deep Learning model.
+
+*(This section will have 4 markdown cells and 15-20 code cells. Total: 20-25 cells).*
+* **NOTE:** All chosen architectures, splitting strategies, and training parameters must be covered in weekly topics. Justify all decisions based on the specific CV problem and data.
+
+---
+
+# 8. Model Evaluation (First-level heading markdown)
+* Rigorously evaluate the performance of your trained Computer Vision model(s).
+* It is crucial to **deeply analyze the problem statement and objective** to select the correct evaluation metrics that directly align with the business goal.
+* Each distinct evaluation operation must be performed in its own separate code cell.
+
+## Metric Selection and Justification (Second-level heading markdown)
+* **Think and decide** on the most appropriate evaluation metric(s) for your CV task.
+* **Examples of metrics:**
+    * **Image Classification:** Accuracy, Precision, Recall, F1-score (macro/micro/weighted), ROC-AUC.
+    * **Image Segmentation:** Intersection over Union (IoU), Dice Coefficient.
+    * **Object Detection:** Mean Average Precision (mAP), IoU for bounding box accuracy.
+* Justify your choice of metrics.
+* **Code Cell:** Display and explain the chosen metrics.
+
+## Model Performance Evaluation (Second-level heading markdown)
+* Evaluate the trained CV model(s) on the unseen test set using the selected metrics.
+* **Examples of visualizations:**
+    * Plotting training/validation loss/metric curves over epochs.
+    * Displaying confusion matrix for classification.
+    * Visualizing sample predictions with overlaid bounding boxes (detection) or masks (segmentation).
+* **Code Cell(s):** Make predictions on the test set, calculate metrics, and include code for any relevant visualizations.
+
+## Final Best Model Selection (Second-level heading markdown)
+* If multiple models/configurations were trained, **think, analyze, and decide** on criteria to select the best one.
+* Compare performance across chosen metrics.
+* **Code Cell:** Present comparison results clearly and explicitly identify the final best model (and optionally save it).
+
+*(This section will have 4 markdown cells and 3-6 code cells. Total: 7-10 cells).*
+* **NOTE:** All evaluation methods and metrics must be covered in weekly topics.
+""",
+
+        "natural_language_processing": """
+1. **Problem Statement**
+   - Business Context (markdown)
+   - Objective of using NLP (e.g., sentiment analysis, text classification) (markdown)
+   - Dataset Overview (markdown)
+   - Data Dictionary or description of text fields and labels (markdown)
+
+2. **Setup: Imports & Configuration**
+   - Import libraries (NLTK, spaCy, scikit-learn, Hugging Face Transformers, etc.) (code)
+   - Set random seeds and config (code)
+
+3. **Data Overview**
+   - Load the dataset and preview sample text (code)
+   - Check text lengths, class distribution (if classification) (code)
+   - Nulls, duplicates, basic stats (code)
+
+4. **Text Preprocessing**
+   - Clean text: lowercasing, punctuation removal, stopwords removal (code)
+   - Tokenization, lemmatization/stemming (code)
+   - Optional: POS tagging, Named Entity Recognition (code)
+   - Convert to numerical form (TF-IDF, CountVectorizer, or token ids) (code)
+
+5. **Modeling**
+   - Classical ML pipeline (Logistic Regression, SVM, Naive Bayes with vectorized text) (code)
+   - OR Transformer-based modeling (BERT/DistilBERT with Hugging Face) (code)
+   - Model training and evaluation with suitable metrics (e.g., F1, accuracy, AUC) (code)
+
+6. **Evaluation & Interpretation**
+   - Classification report, confusion matrix (code)
+   - Visualize word importance or attention weights (code)
+   - Analyze misclassifications or hard examples (code)
+
+7. **Insights & Recommendations**
+   - Summary of results and learnings (markdown)
+   - Recommendations for improving NLP pipeline or model (markdown)
+   - Optional: discussion of domain-specific language issues or future improvements (markdown)
+""",
+
+        "generative_ai_prompt_engineering": """
+1. **Problem Statement**
+   - Use-case Context (markdown)
+   - Goal of prompting (e.g., creative generation, question answering) (markdown)
+   - Dataset or API description if relevant (markdown)
+
+2. **Setup: Imports & Configuration**
+   - Install and import LLM client libraries (OpenAI SDK, LangChain, Transformers) (code)
+   - Configure API keys, endpoints, or local model paths (code)
+
+3. **Prompt Fundamentals**
+   - Explain prompt components: system vs user vs few‐shot examples (markdown)
+   - Show simple zero-shot example (code)
+
+4. **Prompt Design Patterns**
+   - **Few-Shot**: crafting examples in prompt (code + markdown)
+   - **Chain-of-Thought**: breaking reasoning steps (code + markdown)
+   - **Tree-of-Thought**: exploring multiple reasoning branches (code + markdown)
+   - **ReAct**: tool-calling patterns (code + markdown)
+   - **Template Engineering**: building reusable prompt templates (code + markdown)
+
+5. **Evaluation & Iteration**
+   - Metrics: AI-score, human eval, automatic eval (BLEU/ROUGE) (markdown)
+   - Test on edge cases and adversarial examples (code + markdown)
+   - Automated prompt optimization loops (e.g., prompt tuning via LLM) (code + markdown)
+
+6. **Best Practices & Pitfalls**
+   - Token budget management and cost awareness (markdown)
+   - Hallucination mitigation techniques (markdown)
+   - Ethical considerations (bias, privacy) (markdown)
+
+7. **Insights & Recommendations**
+   - Summary of what worked, what didn’t (markdown)
+   - Next steps: chain refinement, retrieval augmentation, or finetuning (markdown)
+""",
+        "generative_ai_rag": """
+1. **Problem Statement**
+   - Use-case Context (markdown)
+   - Objective of RAG solution (e.g., knowledge retrieval + LLM answer) (markdown)
+   - Source data description (documents, PDFs, web pages) (markdown)
+
+2. **Setup: Imports & Configuration**
+   - Install and import libraries (LangChain, OpenAI SDK, FAISS/Chroma, etc.) (code)
+   - Configure API keys or local model paths (code)
+
+3. **Data Ingestion & Indexing**
+   - Load raw documents (text, PDFs, HTML) (code)
+   - Chunking & text splitting strategy (code)
+   - Compute embeddings (OpenAI, Hugging Face) (code)
+   - Build vector store (FAISS, ChromaDB) (code)
+
+4. **Retriever Configuration**
+   - Define similarity search parameters (k, distance metric) (code)
+   - Set up hybrid retrievers (keyword + semantic) if needed (code)
+
+5. **LLM & Chain Assembly**
+   - Create prompt templates for retrieved context + query (code)
+   - Instantiate RetrievalQA or ConversationalRetrievalChain (code)
+   - Configure streaming vs. non-streaming responses
+
+6. **Query Handling & Response Generation**
+   - Test with example queries (code)
+   - Display retrieved passages alongside answers (code + markdown)
+   - Handle edge cases (no hits, multi-document answers)
+
+7. **Evaluation & Metrics**
+   - Measure retrieval quality (recall@k, precision@k) (code)
+   - Evaluate answer quality (ROUGE, human feedback) (markdown)
+   - Logging and traceability of sources
+
+8. **Insights & Recommendations**
+   - Analysis of retrieval/answer performance (markdown)
+   - Suggestions for improving chunk size, embeddings, or model prompts (markdown)
+""",
+
+        "generative_ai_fine_tuning": """
+1. **Problem Statement**
+   - Use-case Context (markdown)
+   - Why fine-tuning is needed (markdown)
+   - Description of base model and target behavior (markdown)
+
+2. **Setup: Imports & Configuration**
+   - Install and import fine-tuning libraries (OpenAI CLI/SDK, Hugging Face Transformers, PEFT) (code)
+   - Authenticate and configure API keys or model paths (code)
+
+3. **Data Preparation**
+   - Collect and format training examples (instruction–response pairs) (code)
+   - Clean and validate dataset (remove duplicates, ensure token limits) (code)
+   - Split into train/validation sets (code)
+
+4. **Model & Training Configuration**
+   - Choose fine-tuning method (e.g., full-model, LoRA/PEFT, adapter) (markdown)
+   - Define hyperparameters: learning rate, batch size, epochs (code or markdown)
+   - Set up training callbacks (early stopping, checkpointing) (code)
+
+5. **Fine-Tuning Execution**
+   - Launch fine-tuning job via API or local script (code)
+   - Monitor training logs (loss curves, metrics) (code)
+
+6. **Evaluation & Validation**
+   - Evaluate on held-out validation data (perplexity, accuracy) (code)
+   - Human-in-the-loop sample reviews (markdown)
+   - Compare to base model on key tasks (markdown + code)
+
+7. **Deployment & Inference**
+   - Save and load the fine-tuned model (code)
+   - Build simple inference pipeline (API, CLI, or notebook demo) (code)
+
+8. **Insights & Recommendations**
+   - Summary of fine-tuning performance (markdown)
+   - Lessons learned on data quality, hyperparameters (markdown)
+   - Next steps: further tuning, active learning, broader use cases (markdown)
+"""
+    }
+
+    domain_key = domain.lower().replace(" ", "_")
+    blueprint = domain_blueprints.get(domain_key)
+
+    if blueprint is None:
+        raise ValueError(f"No blueprint found for domain: {domain}")
+
+    return blueprint.strip()
+
+
+
+# Utility function to classify plot vs. non-plot code
+def is_plot_code(code: str) -> bool:
+    """Return True if code likely generates a plot (matplotlib/seaborn)."""
+    return bool(re.search(r"(plt\.|sns\.|figure\()", code))
+
+
+
+
+# Utility to stage data/dependency files into a working directory
+def stage_dependencies(dataset_path: str, work_dir: str):
+    """
+    Copy the main dataset (and any supporting files) into a clean scratch directory
+    so that repl‑based re‑execution (for plot analysis) can import them.
+    """
+    os.makedirs(work_dir, exist_ok=True)
+    if dataset_path and os.path.isfile(dataset_path):
+        shutil.copy(dataset_path, os.path.join(work_dir, os.path.basename(dataset_path)))
+
+
+
+
+def build_plot_insight_agent_executor(model) -> AgentExecutor:
+    python_repl = PythonREPL()
+    repl_tool = Tool(
+        name="python_repl",
+        description=(
+            "Executes Python code (including plotting) and returns either "
+            "printed statistics or textual description of the plot object."
+        ),
+        func=python_repl.run,
+    )
+
+    # 1) Wrap your system instructions
+    system_template = SystemMessagePromptTemplate.from_template(
+        """You are a Python data visualization specialist. When given Python code
+including data loading and plotting commands, you will:
+
+1. Execute the code via the python_repl tool.
+2. Inspect the resulting Matplotlib/Seaborn figure or printed stats.
+3. Return a concise, bullet-point insight describing:
+   - What the plot shows (trends, distributions, anomalies).
+   - Why those features matter for the business problem.
+"""
+    )
+
+    # 3) Build a ChatPromptTemplate from those
+    chat_prompt = ChatPromptTemplate.from_messages([
+        system_template,
+        ("human", "{input}"),
+        ("placeholder", "{agent_scratchpad}"),
+    ])
+
+    # 4) Create the agent with the proper prompt object
+    agent = create_tool_calling_agent(
+        llm=model,
+        tools=[repl_tool],
+        prompt=chat_prompt
+    )
+
+    executor = AgentExecutor(agent=agent, tools=[repl_tool], verbose=False)
+    return executor