Update app.py

app.py

@@ -1,615 +1,396 @@
-import gradio as gr
+import os
+import torch
 import json
-import re
-import random
 import time
-import os
-from transformers import pipeline
-from huggingface_hub import HfApi
+import logging
+from datetime import datetime
+from threading import Thread
+from queue import Queue
+from transformers import AutoTokenizer, AutoModelForCausalLM
 
-# Set constants
-DEFAULT_NUM_QUESTIONS = 3
-DEFAULT_DIFFICULTY = "Medium"
-MODEL_GENERATION = "facebook/opt-1.3b"  # Free model for question generation
-MODEL_VERIFICATION = "gpt2-large"       # Free model for verification
+# Configuration
+PRIMARY_MODEL = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"  # First model to try
+SECONDARY_MODEL = "facebook/opt-1.3b"  # More powerful backup model
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+BATCH_SIZE = 5  # Process 5 chapters at a time
+MAX_RETRIES = 3
+OUTPUT_DIR = "calculus_textbook_output"
+LOG_FILE = "textbook_generation.log"
 
-# Initialize models (with low memory footprint)
-try:
-    question_generator = pipeline("text-generation", model=MODEL_GENERATION, max_length=1000)
-    question_verifier = pipeline("text-generation", model=MODEL_VERIFICATION, max_length=300)
-except Exception as e:
-    print(f"Model loading error: {str(e)}. Will attempt to load on first use.")
-    question_generator = None
-    question_verifier = None
+# Setup logging
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+logging.basicConfig(
+    filename=os.path.join(OUTPUT_DIR, LOG_FILE),
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
 
-# Calculus curriculum from James Stewart's textbooks
-calculus_curriculum = [
-    {
-        "chapter": "1. Functions and Models",
-        "subchapters": [
-            "1.1 Four Ways to Represent a Function",
-            "1.2 Mathematical Models",
-            "1.3 New Functions from Old Functions",
-            "1.4 Exponential Functions",
-            "1.5 Inverse Functions and Logarithms",
-            "1.6 Parametric Curves"
-        ],
-        "key_formulas": [
-            "Domain and Range",
-            "Function composition: $(f \\circ g)(x) = f(g(x))$",
-            "Exponential function: $f(x) = a^x$, where $a > 0$",
-            "Natural exponential function: $f(x) = e^x$",
-            "Logarithmic function: $f(x) = \\log_a(x)$, where $a > 0, a \\neq 1$",
-            "Natural logarithm: $f(x) = \\ln(x)$"
-        ]
-    },
-    {
-        "chapter": "2. Limits and Derivatives",
-        "subchapters": [
-            "2.1 The Tangent and Velocity Problems",
-            "2.2 The Limit of a Function",
-            "2.3 Calculating Limits",
-            "2.4 Continuity",
-            "2.5 The Derivative",
-            "2.6 The Derivative as a Function",
-            "2.7 Derivatives of Trigonometric Functions",
-            "2.8 The Chain Rule",
-            "2.9 Implicit Differentiation",
-            "2.10 Related Rates",
-            "2.11 Linear Approximations and Differentials"
-        ],
-        "key_formulas": [
-            "Limit Definition: $\\lim_{x \\to a} f(x) = L$",
-            "Derivative Definition: $f'(x) = \\lim_{h \\to 0} \\frac{f(x+h) - f(x)}{h}$",
-            "Power Rule: $\\frac{d}{dx}(x^n) = nx^{n-1}$",
-            "Product Rule: $\\frac{d}{dx}[f(x)g(x)] = f'(x)g(x) + f(x)g'(x)$",
-            "Quotient Rule: $\\frac{d}{dx}\\left[\\frac{f(x)}{g(x)}\\right] = \\frac{f'(x)g(x) - f(x)g'(x)}{[g(x)]^2}$",
-            "Chain Rule: $\\frac{d}{dx}[f(g(x))] = f'(g(x)) \\cdot g'(x)$"
-        ]
-    },
-    {
-        "chapter": "3. Applications of Differentiation",
-        "subchapters": [
-            "3.1 Maximum and Minimum Values",
-            "3.2 The Mean Value Theorem",
-            "3.3 How Derivatives Affect the Shape of a Graph",
-            "3.4 Indeterminate Forms and L'Hospital's Rule",
-            "3.5 Summary of Curve Sketching",
-            "3.6 Optimization Problems",
-            "3.7 Newton's Method",
-            "3.8 Antiderivatives"
-        ],
-        "key_formulas": [
-            "Critical Points: $f'(x) = 0$ or $f'(x)$ is undefined",
-            "Mean Value Theorem: If $f$ is continuous on $[a, b]$ and differentiable on $(a, b)$, then there exists a $c$ in $(a, b)$ such that $f'(c) = \\frac{f(b) - f(a)}{b - a}$",
-            "Second Derivative Test: If $f'(c) = 0$ and $f''(c) > 0$, then $f$ has a local minimum at $c$",
-            "L'Hospital's Rule: $\\lim_{x \\to a}\\frac{f(x)}{g(x)} = \\lim_{x \\to a}\\frac{f'(x)}{g'(x)}$",
-            "Newton's Method: $x_{n+1} = x_n - \\frac{f(x_n)}{f'(x_n)}$"
-        ]
-    },
-    {
-        "chapter": "4. Integrals",
-        "subchapters": [
-            "4.1 Areas and Distances",
-            "4.2 The Definite Integral",
-            "4.3 The Fundamental Theorem of Calculus",
-            "4.4 Indefinite Integrals and the Net Change Theorem",
-            "4.5 The Substitution Rule"
-        ],
-        "key_formulas": [
-            "Definite Integral: $\\int_a^b f(x)\\,dx = \\lim_{n \\to \\infty} \\sum_{i=1}^{n} f(x_i^*)\\Delta x$",
-            "Fundamental Theorem of Calculus: $\\int_a^b f(x)\\,dx = F(b) - F(a)$ where $F'(x) = f(x)$",
-            "Indefinite Integral: $\\int f(x)\\,dx = F(x) + C$ where $F'(x) = f(x)$",
-            "Power Rule for Integration: $\\int x^n\\,dx = \\frac{x^{n+1}}{n+1} + C$ for $n \\neq -1$",
-            "Substitution Rule: $\\int f(g(x))g'(x)\\,dx = \\int f(u)\\,du$ where $u = g(x)$"
-        ]
-    },
-    {
-        "chapter": "5. Applications of Integration",
-        "subchapters": [
-            "5.1 Areas Between Curves",
-            "5.2 Volumes",
-            "5.3 Volumes by Cylindrical Shells",
-            "5.4 Work",
-            "5.5 Average Value of a Function"
-        ],
-        "key_formulas": [
-            "Area Between Curves: $\\int_a^b [f(x) - g(x)]\\,dx$ where $f(x) \\geq g(x)$",
-            "Volume by Disk Method: $V = \\pi\\int_a^b [R(x)]^2\\,dx$",
-            "Volume by Washer Method: $V = \\pi\\int_a^b [(R(x))^2 - (r(x))^2]\\,dx$",
-            "Volume by Cylindrical Shells: $V = 2\\pi\\int_a^b xf(x)\\,dx$ for rotation about y-axis",
-            "Average Value of $f$ on $[a,b]$: $f_{avg} = \\frac{1}{b-a}\\int_a^b f(x)\\,dx$",
-            "Work: $W = \\int_a^b F(x)\\,dx$"
-        ]
-    },
-    {
-        "chapter": "6. Techniques of Integration",
-        "subchapters": [
-            "6.1 Integration by Parts",
-            "6.2 Trigonometric Integrals",
-            "6.3 Trigonometric Substitution",
-            "6.4 Integration of Rational Functions by Partial Fractions",
-            "6.5 Strategy for Integration",
-            "6.6 Approximate Integration",
-            "6.7 Improper Integrals"
-        ],
-        "key_formulas": [
-            "Integration by Parts: $\\int u\\,dv = uv - \\int v\\,du$",
-            "Trigonometric Integrals: $\\int \\sin^n x \\cos^m x\\,dx$ (various formulas)",
-            "Trig Substitution: $x = a\\sin\\theta$ for $\\sqrt{a^2-x^2}$, $x = a\\tan\\theta$ for $\\sqrt{a^2+x^2}$",
-            "Partial Fractions: $\\frac{P(x)}{Q(x)} = \\frac{A}{(x-a)} + \\frac{B}{(x-a)^2} + \\frac{Cx+D}{x^2+bx+c} + ...$",
-            "Improper Integrals: $\\int_a^{\\infty} f(x)\\,dx = \\lim_{t \\to \\infty} \\int_a^t f(x)\\,dx$"
-        ]
-    },
-    {
-        "chapter": "7. Differential Equations",
-        "subchapters": [
-            "7.1 Modeling with Differential Equations",
-            "7.2 Direction Fields and Euler's Method",
-            "7.3 Separable Equations",
-            "7.4 Models for Population Growth",
-            "7.5 Linear Equations",
-            "7.6 Predator-Prey Systems"
-        ],
-        "key_formulas": [
-            "General form of a first-order differential equation: $\\frac{dy}{dx} = f(x, y)$",
-            "Separable equation: $\\frac{dy}{dx} = g(x)h(y)$ → $\\int \\frac{1}{h(y)}dy = \\int g(x)dx + C$",
-            "First-order linear differential equation: $\\frac{dy}{dx} + P(x)y = Q(x)$",
-            "Integrating factor method: Multiply by $e^{\\int P(x)dx}$",
-            "Euler's Method: $y_{n+1} = y_n + hf(x_n, y_n)$"
-        ]
-    },
-    {
-        "chapter": "8. Infinite Sequences and Series",
-        "subchapters": [
-            "8.1 Sequences",
-            "8.2 Series",
-            "8.3 The Integral Test and Estimates of Sums",
-            "8.4 The Comparison Tests",
-            "8.5 Alternating Series",
-            "8.6 Absolute Convergence and the Ratio and Root Tests",
-            "8.7 Strategy for Testing Series",
-            "8.8 Power Series",
-            "8.9 Representations of Functions as Power Series",
-            "8.10 Taylor and Maclaurin Series"
-        ],
-        "key_formulas": [
-            "Geometric Series: $\\sum_{n=0}^{\\infty} ar^n = \\frac{a}{1-r}$ if $|r| < 1$",
-            "Taylor Series: $f(x) = \\sum_{n=0}^{\\infty} \\frac{f^{(n)}(a)}{n!}(x-a)^n$",
-            "Maclaurin Series: $f(x) = \\sum_{n=0}^{\\infty} \\frac{f^{(n)}(0)}{n!}x^n$",
-            "Common Maclaurin Series: $e^x = \\sum_{n=0}^{\\infty} \\frac{x^n}{n!}$, $\\sin(x) = \\sum_{n=0}^{\\infty} \\frac{(-1)^n}{(2n+1)!}x^{2n+1}$",
-            "Ratio Test: $\\lim_{n \\to \\infty} |\\frac{a_{n+1}}{a_n}| < 1$ implies convergence"
-        ]
-    },
-    {
-        "chapter": "9. Parametric Equations and Polar Coordinates",
-        "subchapters": [
-            "9.1 Parametric Curves",
-            "9.2 Calculus with Parametric Curves",
-            "9.3 Polar Coordinates",
-            "9.4 Areas and Lengths in Polar Coordinates",
-            "9.5 Conic Sections"
-        ],
-        "key_formulas": [
-            "Parametric curve: $x = f(t)$, $y = g(t)$",
-            "Arc length of parametric curve: $L = \\int_a^b \\sqrt{[f'(t)]^2 + [g'(t)]^2}\\,dt$",
-            "Polar to rectangular coordinates: $x = r\\cos\\theta$, $y = r\\sin\\theta$",
-            "Rectangular to polar coordinates: $r = \\sqrt{x^2 + y^2}$, $\\theta = \\arctan(\\frac{y}{x})$",
-            "Area in polar coordinates: $A = \\frac{1}{2}\\int_{\\alpha}^{\\beta} [r(\\theta)]^2\\,d\\theta$"
-        ]
-    },
-    {
-        "chapter": "10. Vectors and the Geometry of Space",
-        "subchapters": [
-            "10.1 Three-Dimensional Coordinate Systems",
-            "10.2 Vectors",
-            "10.3 The Dot Product",
-            "10.4 The Cross Product",
-            "10.5 Equations of Lines and Planes",
-            "10.6 Cylinders and Quadric Surfaces"
-        ],
-        "key_formulas": [
-            "Dot Product: $\\vec{a} \\cdot \\vec{b} = |\\vec{a}||\\vec{b}|\\cos\\theta$",
-            "Cross Product: $\\vec{a} \\times \\vec{b} = |\\vec{a}||\\vec{b}|\\sin\\theta\\,\\vec{n}$",
-            "Equation of a line: $\\vec{r} = \\vec{r_0} + t\\vec{v}$",
-            "Equation of a plane: $\\vec{n} \\cdot (\\vec{r} - \\vec{r_0}) = 0$ or $ax + by + cz + d = 0$",
-            "Distance from point to plane: $d = \\frac{|ax_0 + by_0 + cz_0 + d|}{\\sqrt{a^2 + b^2 + c^2}}$"
-        ]
-    },
-    {
-        "chapter": "11. Vector Functions",
-        "subchapters": [
-            "11.1 Vector Functions and Space Curves",
-            "11.2 Derivatives and Integrals of Vector Functions",
-            "11.3 Arc Length and Curvature",
-            "11.4 Motion in Space: Velocity and Acceleration"
-        ],
-        "key_formulas": [
-            "Vector function: $\\vec{r}(t) = x(t)\\vec{i} + y(t)\\vec{j} + z(t)\\vec{k}$",
-            "Derivative of vector function: $\\vec{r}'(t) = x'(t)\\vec{i} + y'(t)\\vec{j} + z'(t)\\vec{k}$",
-            "Arc length: $L = \\int_a^b |\\vec{r}'(t)|\\,dt$",
-            "Unit tangent vector: $\\vec{T}(t) = \\frac{\\vec{r}'(t)}{|\\vec{r}'(t)|}$",
-            "Curvature: $\\kappa = \\frac{|\\vec{T}'(t)|}{|\\vec{r}'(t)|}$",
-            "Acceleration: $\\vec{a}(t) = \\vec{r}''(t)$"
-        ]
-    },
-    {
-        "chapter": "12. Partial Derivatives",
-        "subchapters": [
-            "12.1 Functions of Several Variables",
-            "12.2 Limits and Continuity",
-            "12.3 Partial Derivatives",
-            "12.4 Tangent Planes and Linear Approximations",
-            "12.5 The Chain Rule",
-            "12.6 Directional Derivatives and the Gradient Vector",
-            "12.7 Maximum and Minimum Values",
-            "12.8 Lagrange Multipliers"
-        ],
-        "key_formulas": [
-            "Partial derivative: $\\frac{\\partial f}{\\partial x}(x_0, y_0)$",
-            "Gradient: $\\nabla f = \\frac{\\partial f}{\\partial x}\\vec{i} + \\frac{\\partial f}{\\partial y}\\vec{j} + \\frac{\\partial f}{\\partial z}\\vec{k}$",
-            "Directional derivative: $D_\\vec{u}f = \\nabla f \\cdot \\vec{u}$",
-            "Tangent plane: $z - z_0 = f_x(x_0, y_0)(x - x_0) + f_y(x_0, y_0)(y - y_0)$",
-            "Chain Rule: $\\frac{dz}{dt} = \\frac{\\partial z}{\\partial x}\\frac{dx}{dt} + \\frac{\\partial z}{\\partial y}\\frac{dy}{dt}$"
-        ]
-    },
-    {
-        "chapter": "13. Multiple Integrals",
-        "subchapters": [
-            "13.1 Double Integrals over Rectangles",
-            "13.2 Iterated Integrals",
-            "13.3 Double Integrals over General Regions",
-            "13.4 Double Integrals in Polar Coordinates",
-            "13.5 Applications of Double Integrals",
-            "13.6 Triple Integrals",
-            "13.7 Triple Integrals in Cylindrical Coordinates",
-            "13.8 Triple Integrals in Spherical Coordinates",
-            "13.9 Change of Variables in Multiple Integrals"
-        ],
-        "key_formulas": [
-            "Double integral: $\\iint_R f(x,y)\\,dA = \\int_a^b \\int_c^d f(x,y)\\,dy\\,dx$",
-            "Area in polar coordinates: $\\iint_R f(r,\\theta)\\,dA = \\int_{\\alpha}^{\\beta} \\int_{h_1(\\theta)}^{h_2(\\theta)} f(r,\\theta)\\,r\\,dr\\,d\\theta$",
-            "Triple integral: $\\iiint_E f(x,y,z)\\,dV$",
-            "Cylindrical coordinates: $\\iiint_E f(x,y,z)\\,dV = \\iiint_E f(r\\cos\\theta, r\\sin\\theta, z)\\,r\\,dr\\,d\\theta\\,dz$",
-            "Spherical coordinates: $\\iiint_E f(x,y,z)\\,dV = \\iiint_E f(\\rho\\sin\\phi\\cos\\theta, \\rho\\sin\\phi\\sin\\theta, \\rho\\cos\\phi)\\,\\rho^2\\sin\\phi\\,d\\rho\\,d\\phi\\,d\\theta$"
-        ]
-    },
-    {
-        "chapter": "14. Vector Calculus",
-        "subchapters": [
-            "14.1 Vector Fields",
-            "14.2 Line Integrals",
-            "14.3 The Fundamental Theorem for Line Integrals",
-            "14.4 Green's Theorem",
-            "14.5 Curl and Divergence",
-            "14.6 Surface Integrals",
-            "14.7 Stokes' Theorem",
-            "14.8 The Divergence Theorem"
-        ],
-        "key_formulas": [
-            "Line integral of scalar function: $\\int_C f(x,y,z)\\,ds = \\int_a^b f(\\vec{r}(t))|\\vec{r}'(t)|\\,dt$",
-            "Line integral of vector field: $\\int_C \\vec{F} \\cdot d\\vec{r} = \\int_a^b \\vec{F}(\\vec{r}(t)) \\cdot \\vec{r}'(t)\\,dt$",
-            "Green's Theorem: $\\oint_C (P\\,dx + Q\\,dy) = \\iint_D (\\frac{\\partial Q}{\\partial x} - \\frac{\\partial P}{\\partial y})\\,dA$",
-            "Divergence: $\\text{div}\\,\\vec{F} = \\nabla \\cdot \\vec{F} = \\frac{\\partial P}{\\partial x} + \\frac{\\partial Q}{\\partial y} + \\frac{\\partial R}{\\partial z}$",
-            "Curl: $\\text{curl}\\,\\vec{F} = \\nabla \\times \\vec{F}$",
-            "Stokes' Theorem: $\\int_S (\\nabla \\times \\vec{F}) \\cdot d\\vec{S} = \\oint_C \\vec{F} \\cdot d\\vec{r}$",
-            "Divergence Theorem: $\\iiint_E (\\nabla \\cdot \\vec{F})\\,dV = \\iint_{\\partial E} \\vec{F} \\cdot d\\vec{S}$"
-        ]
-    }
-]
-
-def load_models_if_needed():
-    """Ensures models are loaded when needed"""
-    global question_generator, question_verifier
+class ModelManager:
+    """Manages loading and switching between language models for text generation."""
     
-    if question_generator is None:
-        try:
-            question_generator = pipeline("text-generation", model=MODEL_GENERATION, max_length=1000)
-        except Exception as e:
-            return f"Error loading question generator: {str(e)}"
+    def __init__(self):
+        self.models = {}
+        self.tokenizers = {}
+        self.current_model = None
+        
+    def load_model(self, model_name):
+        """Load a model and its tokenizer if not already loaded."""
+        if model_name not in self.models:
+            try:
+                logging.info(f"Loading model: {model_name}")
+                tokenizer = AutoTokenizer.from_pretrained(model_name)
+                model = AutoModelForCausalLM.from_pretrained(
+                    model_name,
+                    torch_dtype=torch.float16 if DEVICE == "cuda" else torch.float32,
+                    device_map="auto" if DEVICE == "cuda" else None
+                )
+                model.eval()
+                
+                self.models[model_name] = model
+                self.tokenizers[model_name] = tokenizer
+                logging.info(f"Successfully loaded model: {model_name}")
+                return True
+            except Exception as e:
+                logging.error(f"Failed to load model {model_name}: {str(e)}")
+                return False
+        return True
     
-    if question_verifier is None:
-        try:
-            question_verifier = pipeline("text-generation", model=MODEL_VERIFICATION, max_length=300)
-        except Exception as e:
-            return f"Error loading question verifier: {str(e)}"
+    def set_current_model(self, model_name):
+        """Set the current model to use for generation."""
+        if model_name not in self.models and not self.load_model(model_name):
+            return False
+        self.current_model = model_name
+        return True
     
-    return None
+    def generate_text(self, prompt, max_length=1024):
+        """Generate text using the current model."""
+        if not self.current_model:
+            raise ValueError("No model selected. Call set_current_model first.")
+        
+        model = self.models[self.current_model]
+        tokenizer = self.tokenizers[self.current_model]
+        
+        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+        
+        # Generate with some randomness for creativity
+        with torch.no_grad():
+            outputs = model.generate(
+                **inputs,
+                max_length=max_length,
+                temperature=0.7,
+                top_p=0.9,
+                do_sample=True,
+                pad_token_id=tokenizer.eos_token_id
+            )
+            
+        response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        # Extract only the generated part
+        generated_text = response[len(tokenizer.decode(inputs['input_ids'][0], skip_special_tokens=True)):].strip()
+        return generated_text
 
-def get_chapter_summary(chapter_idx, subchapter_idx=None):
-    """Get summary of selected chapter and subchapter"""
-    if chapter_idx < 0 or chapter_idx >= len(calculus_curriculum):
-        return "Invalid chapter selection."
-    
-    chapter = calculus_curriculum[chapter_idx]
+class CalculusTextbookGenerator:
+    """Generates a complete calculus textbook with questions and solutions."""
     
-    if subchapter_idx is None or subchapter_idx < 0 or subchapter_idx >= len(chapter["subchapters"]):
-        # Return chapter summary only
-        summary = f"# {chapter['chapter']}\n\n"
-        summary += "## Key Formulas\n"
-        for formula in chapter.get("key_formulas", []):
-            summary += f"- {formula}\n"
-        return summary
-    
-    # Return specific subchapter
-    subchapter = chapter["subchapters"][subchapter_idx]
-    summary = f"# {chapter['chapter']}\n## {subchapter}\n\n"
-    summary += "### Key Formulas\n"
-    for formula in chapter.get("key_formulas", []):
-        summary += f"- {formula}\n"
-    
-    return summary
+    def __init__(self):
+        self.model_manager = ModelManager()
+        self.textbook_data = self.create_initial_textbook_structure()
+        
+    def create_initial_textbook_structure(self):
+        """Create the initial structure of the calculus textbook."""
+        return {
+            "books": [
+                {
+                    "name": "Calculus 1: Early Transcendentals",
+                    "details": "Introduction to single-variable calculus including limits, derivatives, and basic integration techniques.",
+                    "chapters": [
+                        {
+                            "chapterTitle": "Chapter 6: Applications of Integration",
+                            "subChapters": [
+                                "6.1: Areas Between Curves", 
+                                "6.2: Volumes", 
+                                "6.3: Volumes by Cylindrical Shells",
+                                "6.4: Work",
+                                "6.5: Average Value of a Function"
+                            ],
+                            "questions": []  # Will be filled with generated questions
+                        },
+                        {
+                            "chapterTitle": "Chapter 8: Further Applications of Integration",
+                            "subChapters": [
+                                "8.1: Arc Length",
+                                "8.2: Area of a Surface of Revolution",
+                                "8.3: Applications to Physics and Engineering",
+                                "8.4: Applications to Economics and Biology",
+                                "8.5: Probability"
+                            ],
+                            "questions": []
+                        },
+                        {
+                            "chapterTitle": "Chapter 9: Differential Equations",
+                            "subChapters": [
+                                "9.1: Modeling with Differential Equations",
+                                "9.2: Direction Fields and Euler's Method",
+                                "9.3: Separable Equations",
+                                "9.4: Models for Population Growth",
+                                "9.5: Linear Equations",
+                                "9.6: Predator–Prey Systems"
+                            ],
+                            "questions": []
+                        },
+                        {
+                            "chapterTitle": "Chapter 10: Parametric Equations and Polar Coordinates",
+                            "subChapters": [
+                                "10.1: Curves Defined by Parametric Equations",
+                                "10.2: Calculus with Parametric Curves",
+                                "10.3: Polar Coordinates",
+                                "10.4: Calculus in Polar Coordinates",
+                                "10.5: Conic Sections",
+                                "10.6: Conic Sections in Polar Coordinates"
+                            ],
+                            "questions": []
+                        },
+                        {
+                            "chapterTitle": "Chapter 11: Sequences, Series, and Power Series",
+                            "subChapters": [
+                                "11.1: Sequences",
+                                "11.2: Series",
+                                "11.3: The Integral Test and Estimates of Sums",
+                                "11.4: The Comparison Tests",
+                                "11.5: Alternating Series and Absolute Convergence",
+                                "11.6: The Ratio and Root Tests",
+                                "11.7: Power Series"
+                            ],
+                            "questions": []
+                        }
+                    ]
+                },
+                {
+                    "name": "Calculus 2: Advanced Concepts",
+                    "details": "Advances into series, sequences, techniques of integration, and vector calculus.",
+                    "chapters": [
+                        {
+                            "chapterTitle": "Chapter 12: Vectors and the Geometry of Space",
+                            "subChapters": [
+                                "12.1: Three-Dimensional Coordinate Systems",
+                                "12.2: Vectors",
+                                "12.3: The Dot Product",
+                                "12.4: The Cross Product",
+                                "12.5: Equations of Lines and Planes",
+                                "12.6: Cylinders and Quadric Surfaces"
+                            ],
+                            "questions": []
+                        },
+                        {
+                            "chapterTitle": "Chapter 13: Vector Functions",
+                            "subChapters": [
+                                "13.1: Vector Functions and Space Curves",
+                                "13.2: Derivatives and Integrals of Vector Functions",
+                                "13.3: Arc Length and Curvature",
+                                "13.4: Motion in Space: Velocity and Acceleration"
+                            ],
+                            "questions": []
+                        },
+                        {
+                            "chapterTitle": "Chapter 14: Partial Derivatives",
+                            "subChapters": [
+                                "14.1: Functions of Several Variables",
+                                "14.2: Limits and Continuity",
+                                "14.3: Partial Derivatives",
+                                "14.4: Tangent Planes and Linear Approximation",
+                                "14.5: The Chain Rule"
+                            ],
+                            "questions": []
+                        }
+                    ]
+                }
+            ]
+        }
+        
+    def generate_question_set(self, chapter_title, subchapter_titles, num_questions=3):
+        """Generate a set of questions with step-by-step solutions for a chapter."""
+        
+        # Try the primary model first
+        self.model_manager.set_current_model(PRIMARY_MODEL)
+        
+        prompt = f"""Create {num_questions} calculus questions with detailed step-by-step solutions for:
+{chapter_title}
 
-def generate_question_prompt(chapter, subchapter, difficulty, num_questions=3):
-    """Generate a prompt for the model to create questions"""
-    prompt = f"""Create {num_questions} university-level mathematics questions about {subchapter} from {chapter} at {difficulty} difficulty.
+The questions should cover these subchapters:
+{', '.join(subchapter_titles)}
 
 For each question:
-1. Write a clear, university-level calculus problem that requires understanding of the concepts and techniques.
-2. Include a step-by-step solution showing all work and mathematical reasoning.
-3. Provide the final answer.
-
-Format your response exactly as follows:
+1. Write a clear, university-level calculus problem
+2. Provide a comprehensive step-by-step solution with all math steps shown
+3. Include a final answer
 
-## Question 1
-[Question text]
-
-### Solution
-Step 1: [First step of solution]
-Step 2: [Second step]
-...
-
-### Answer
-[Final answer]
-
-## Question 2
+Format each question as:
+QUESTION: [Problem statement]
+SOLUTION:
+Step 1: [First step with explanation]
+Step 2: [Next step]
 ...
+Final Answer: [The solution]
 
-Make sure all mathematics is correct and your solution steps are clear and logical.
+Make sure to use proper mathematical notation and include a variety of question types.
 """
-    return prompt
-
-def verify_question(question_text, solution_text):
-    """Verify if the question and solution are mathematically sound"""
-    error_msg = load_models_if_needed()
-    if error_msg:
-        return False, error_msg
-    
-    verification_prompt = f"""Verify if this calculus question and solution are mathematically correct:
-    
-Question: {question_text}
 
-Solution: {solution_text}
-
-Is the solution mathematically correct? Answer Yes or No and briefly explain why."""
-
-    try:
-        # Get verification response
-        verification = question_verifier(verification_prompt, max_length=300)[0]['generated_text']
+        try:
+            generated_content = self.model_manager.generate_text(prompt, max_length=2048)
+            
+            # Check if the content looks good
+            if len(generated_content) < 200 or "QUESTION" not in generated_content:
+                # Try the secondary model if the primary one gave poor results
+                logging.warning(f"Primary model gave insufficient results for {chapter_title}. Trying secondary model.")
+                self.model_manager.set_current_model(SECONDARY_MODEL)
+                generated_content = self.model_manager.generate_text(prompt, max_length=2048)
+            
+            # Parse the generated content into question objects
+            questions = self.parse_questions(generated_content)
+            
+            if not questions or len(questions) == 0:
+                logging.warning(f"Failed to parse any questions from content for {chapter_title}")
+                return []
+                
+            return questions
+            
+        except Exception as e:
+            logging.error(f"Error generating questions for {chapter_title}: {str(e)}")
+            return []
+            
+    def parse_questions(self, content):
+        """Parse the generated content into structured question objects."""
+        questions = []
         
-        # Check response for indication that the solution is correct
-        if "yes" in verification.lower() and "incorrect" not in verification.lower() and "error" not in verification.lower():
-            return True, "Verification passed"
-        else:
-            # Extract the explanation for why it's incorrect
-            explanation = verification.split("No")[1] if "No" in verification else "Unable to determine specific issue"
-            return False, f"Verification failed: {explanation}"
-    except Exception as e:
-        return False, f"Error during verification: {str(e)}"
-
-def generate_questions(chapter_index, subchapter_index, difficulty, num_questions):
-    """Generate mathematics questions based on chapter/subchapter"""
-    error_msg = load_models_if_needed()
-    if error_msg:
-        return f"## Error Loading Models\n\n{error_msg}\n\nPlease try again later or contact the administrator."
-
-    # Get chapter and subchapter information
-    if chapter_index < 0 or chapter_index >= len(calculus_curriculum):
-        return "Please select a valid chapter."
+        # Split by "QUESTION:" or similar markers
+        parts = content.split("QUESTION:")
         
-    chapter = calculus_curriculum[chapter_index]
-    
-    if subchapter_index < 0 or subchapter_index >= len(chapter["subchapters"]):
-        return "Please select a valid subchapter."
+        for i, part in enumerate(parts):
+            if i == 0:
+                continue  # Skip the first part (before the first QUESTION:)
+                
+            try:
+                # Split into question and solution
+                if "SOLUTION:" in part:
+                    question_text, solution = part.split("SOLUTION:", 1)
+                else:
+                    # Try alternative formats
+                    for marker in ["Solution:", "STEPS:", "Steps:"]:
+                        if marker in part:
+                            question_text, solution = part.split(marker, 1)
+                            break
+                    else:
+                        question_text = part
+                        solution = ""
+                
+                questions.append({
+                    "question": question_text.strip(),
+                    "solution": solution.strip()
+                })
+            except Exception as e:
+                logging.error(f"Error parsing question {i}: {str(e)}")
+                continue
+                
+        return questions
     
-    subchapter = chapter["subchapters"][subchapter_index]
+    def worker_function(self, queue, results):
+        """Worker thread function to process chapters from queue."""
+        while True:
+            item = queue.get()
+            if item is None:  # None signals to exit
+                queue.task_done()
+                break
+            
+            book_idx, chapter_idx, chapter = item
+            chapter_title = chapter["chapterTitle"]
+            subchapters = chapter.get("subChapters", [])
+            
+            logging.info(f"Processing: {chapter_title}")
+            
+            # Try to generate questions with retries
+            for attempt in range(MAX_RETRIES):
+                try:
+                    questions = self.generate_question_set(chapter_title, subchapters, num_questions=4)
+                    if questions:
+                        # Save the questions to the chapter
+                        self.textbook_data["books"][book_idx]["chapters"][chapter_idx]["questions"] = questions
+                        
+                        logging.info(f"✓ Generated {len(questions)} questions for {chapter_title}")
+                        break  # Success, exit retry loop
+                    else:
+                        logging.warning(f"No questions generated for {chapter_title} on attempt {attempt+1}")
+                        
+                except Exception as e:
+                    logging.error(f"Attempt {attempt+1}/{MAX_RETRIES} failed for {chapter_title}: {str(e)}")
+                    time.sleep(2)  # Wait before retrying
+            
+            # Save current state to file
+            self.save_current_state()
+            queue.task_done()
     
-    # Generate prompt for the model
-    prompt = generate_question_prompt(chapter["chapter"], subchapter, difficulty, num_questions)
+    def save_current_state(self):
+        """Save the current state of the textbook generation."""
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        with open(os.path.join(OUTPUT_DIR, f"textbook_state_{timestamp}.json"), "w") as f:
+            json.dump(self.textbook_data, f, indent=2)
+        
+        # Also save to a fixed filename for the latest state
+        with open(os.path.join(OUTPUT_DIR, "textbook_latest.json"), "w") as f:
+            json.dump(self.textbook_data, f, indent=2)
     
-    try:
-        # Generate questions
-        result = question_generator(prompt, max_length=1500, do_sample=True, 
-                                   temperature=0.7, top_p=0.85, num_return_sequences=1)[0]['generated_text']
+    def process_in_batches(self):
+        """Process all chapters in batches."""
+        queue = Queue()
         
-        # Extract generated questions and solutions
-        result = result.replace(prompt, "")
+        # Queue all chapters for processing
+        for book_idx, book in enumerate(self.textbook_data["books"]):
+            for chapter_idx, chapter in enumerate(book["chapters"]):
+                queue.put((book_idx, chapter_idx, chapter))
         
-        # Basic formatting fixes
-        result = re.sub(r'#+\s*Question', '## Question', result)
-        result = re.sub(r'#+\s*Solution', '### Solution', result)
-        result = re.sub(r'#+\s*Answer', '### Answer', result)
+        # Create and start worker thread
+        worker = Thread(target=self.worker_function, args=(queue, None))
+        worker.daemon = True  # Allow the program to exit even if the thread is running
+        worker.start()
         
-        # Check if we got properly formatted output
-        if "## Question" not in result:
-            # Fallback to template questions for the topic
-            result = generate_fallback_questions(chapter["chapter"], subchapter, num_questions)
+        # Process in batches
+        total_chapters = queue.qsize()
+        processed = 0
         
-        # Add chapter summary at the top
-        summary = get_chapter_summary(chapter_index, subchapter_index)
-        result = f"{summary}\n\n# Generated Questions\n\n{result}"
+        while processed < total_chapters:
+            # Wait for the batch to be processed
+            start_size = queue.qsize()
+            batch_size = min(BATCH_SIZE, start_size)
+            
+            logging.info(f"Processing batch of {batch_size} chapters. {start_size} remaining.")
+            
+            # Wait until this batch is done
+            while queue.qsize() > start_size - batch_size:
+                time.sleep(2)
+            
+            processed += batch_size
+            logging.info(f"Batch complete. {processed}/{total_chapters} chapters processed.")
+            
+            # Save current state
+            self.save_current_state()
         
-        return result
+        # Signal worker to exit
+        queue.put(None)
+        worker.join()
         
-    except Exception as e:
-        return f"Error generating questions: {str(e)}\n\nPlease try again or select a different topic."
-
-def generate_fallback_questions(chapter, subchapter, num_questions):
-    """Generate fallback questions when model generation fails"""
-    # Basic templates for different calculus topics
-    if "Limits" in chapter or "Limits" in subchapter:
-        questions = [
-            {
-                "question": "Evaluate the limit: $\\lim_{x \\to 2} \\frac{x^3 - 8}{x - 2}$",
-                "solution": "Step 1: Note that this is an indeterminate form (0/0) when x = 2.\n" +
-                           "Step 2: Factor the numerator: $x^3 - 8 = (x - 2)(x^2 + 2x + 4)$\n" +
-                           "Step 3: Simplify: $\\lim_{x \\to 2} \\frac{(x - 2)(x^2 + 2x + 4)}{x - 2} = \\lim_{x \\to 2} (x^2 + 2x + 4)$\n" +
-                           "Step 4: Evaluate by direct substitution: $2^2 + 2(2) + 4 = 4 + 4 + 4 = 12$",
-                "answer": "12"
-            },
-            {
-                "question": "Find the limit: $\\lim_{x \\to 0} \\frac{\\sin(3x)}{x}$",
-                "solution": "Step 1: Rewrite using the limit property $\\lim_{x \\to 0} \\frac{\\sin x}{x} = 1$\n" +
-                           "Step 2: $\\lim_{x \\to 0} \\frac{\\sin(3x)}{x} = \\lim_{x \\to 0} 3 \\cdot \\frac{\\sin(3x)}{3x}$\n" +
-                           "Step 3: Apply the limit property: $3 \\cdot \\lim_{x \\to 0} \\frac{\\sin(3x)}{3x} = 3 \\cdot 1 = 3$",
-                "answer": "3"
-            }
-        ]
-    elif "Derivative" in chapter or "Derivative" in subchapter:
-        questions = [
-            {
-                "question": "Find the derivative of $f(x) = x^3\\ln(x) - \\frac{x^3}{3}$",
-                "solution": "Step 1: Use the product rule on $x^3\\ln(x)$\n" +
-                           "$\\frac{d}{dx}[x^3\\ln(x)] = x^3 \\cdot \\frac{1}{x} + \\ln(x) \\cdot 3x^2$\n" +
-                           "$= x^2 + 3x^2\\ln(x)$\n" +
-                           "Step 2: Find the derivative of $\\frac{x^3}{3}$\n" +
-                           "$\\frac{d}{dx}[\\frac{x^3}{3}] = \\frac{3x^2}{3} = x^2$\n" +
-                           "Step 3: Combine the results\n" +
-                           "$f'(x) = x^2 + 3x^2\\ln(x) - x^2 = 3x^2\\ln(x)$",
-                "answer": "$f'(x) = 3x^2\\ln(x)$"
-            }
-        ]
-    elif "Integration" in chapter or "Integral" in chapter or "Integration" in subchapter or "Integral" in subchapter:
-        questions = [
-            {
-                "question": "Evaluate the integral: $\\int x^2\\ln(x) dx$",
-                "solution": "Step 1: Use integration by parts with $u = \\ln(x)$ and $dv = x^2 dx$\n" +
-                           "Then $du = \\frac{1}{x}dx$ and $v = \\frac{x^3}{3}$\n" +
-                           "Step 2: Apply the formula $\\int u dv = uv - \\int v du$\n" +
-                           "$\\int x^2\\ln(x) dx = \\ln(x) \\cdot \\frac{x^3}{3} - \\int \\frac{x^3}{3} \\cdot \\frac{1}{x} dx$\n" +
-                           "$= \\frac{x^3\\ln(x)}{3} - \\frac{1}{3}\\int x^2 dx$\n" +
-                           "$= \\frac{x^3\\ln(x)}{3} - \\frac{1}{3} \\cdot \\frac{x^3}{3} + C$\n" +
-                           "$= \\frac{x^3\\ln(x)}{3} - \\frac{x^3}{9} + C$",
-                "answer": "$\\frac{x^3\\ln(x)}{3} - \\frac{x^3}{9} + C$"
-            }
-        ]
-    else:
-        # Generic calculus questions
-        questions = [
-            {
-                "question": "Find the critical points of $f(x) = x^3 - 6x^2 + 12x + 5$ and determine their nature.",
-                "solution": "Step 1: Find the derivative: $f'(x) = 3x^2 - 12x + 12$\n" +
-                           "Step 2: Set $f'(x) = 0$ and solve: $3x^2 - 12x + 12 = 0$\n" +
-                           "Step 3: Simplify: $x^2 - 4x + 4 = 0$\n" +
-                           "Step 4: Factor: $(x - 2)^2 = 0$\n" +
-                           "Step 5: Therefore $x = 2$ is a critical point (with multiplicity 2)\n" +
-                           "Step 6: Find the second derivative: $f''(x) = 6x - 12$\n" +
-                           "Step 7: Evaluate at $x = 2$: $f''(2) = 6(2) - 12 = 0$\n" +
-                           "Step 8: Since $f''(2) = 0$, the second derivative test is inconclusive\n" +
-                           "Step 9: Checking $f'(x)$ around $x = 2$:\n" +
-                           "For $x < 2$, $f'(x) < 0$ and for $x > 2$, $f'(x) > 0$\n" +
-                           "Step 10: Therefore, $x = 2$ is a point of inflection",
-                "answer": "$x = 2$ is a critical point and an inflection point"
-            }
-        ]
-    
-    # Get a random subset of questions or duplicate if we need more
-    result_questions = []
-    for i in range(num_questions):
-        idx = i % len(questions)
-        q = questions[idx]
-        result_questions.append({
-            "id": i+1,
-            "question": q["question"],
-            "solution": q["solution"],
-            "answer": q["answer"]
-        })
-    
-    # Format the output
-    result = ""
-    for q in result_questions:
-        result += f"## Question {q['id']}\n{q['question']}\n\n"
-        result += f"### Solution\n{q['solution']}\n\n"
-        result += f"### Answer\n{q['answer']}\n\n"
-    
-    return result
-
-def on_chapter_change(chapter_index):
-    """Update subchapter dropdown based on selected chapter"""
-    if chapter_index < 0 or chapter_index >= len(calculus_curriculum):
-        return gr.Dropdown(choices=[], value=None)
-    
-    subchapters = calculus_curriculum[chapter_index]["subchapters"]
-    return gr.Dropdown(choices=subchapters, value=subchapters[0] if subchapters else None)
+        # Save final state
+        self.save_current_state()
+        logging.info("All chapters processed. Textbook generation complete.")
 
-def create_interface():
-    """Create the Gradio interface"""
-    # Extract chapter titles for dropdown
-    chapters = [chapter["chapter"] for chapter in calculus_curriculum]
+def main():
+    start_time = datetime.now()
+    logging.info(f"Starting textbook generation at {start_time}")
     
-    with gr.Blocks(title="Calculus Question Generator", theme=gr.themes.Soft()) as demo:
-        gr.Markdown("# 🧮 Calculus Question Generator")
-        gr.Markdown("Generate university-level calculus questions with step-by-step solutions.")
-        
-        with gr.Row():
-            with gr.Column(scale=2):
-                chapter_dropdown = gr.Dropdown(
-                    choices=chapters,
-                    value=chapters[0] if chapters else None,
-                    label="Chapter",
-                    info="Select a chapter from Stewart's Calculus"
-                )
-                
-                subchapter_dropdown = gr.Dropdown(
-                    choices=calculus_curriculum[0]["subchapters"] if calculus_curriculum else [],
-                    value=calculus_curriculum[0]["subchapters"][0] if calculus_curriculum and calculus_curriculum[0]["subchapters"] else None,
-                    label="Subchapter",
-                    info="Select a specific subchapter"
-                )
-                
-                with gr.Row():
-                    num_questions = gr.Slider(
-                        minimum=1,
-                        maximum=5,
-                        value=DEFAULT_NUM_QUESTIONS,
-                        step=1,
-                        label="Number of Questions"
-                    )
-                    
-                    difficulty = gr.Dropdown(
-                        choices=["Easy", "Medium", "Hard", "Advanced"],
-                        value=DEFAULT_DIFFICULTY,
-                        label="Difficulty Level"
-                    )
-                
-                generate_button = gr.Button("Generate Questions", variant="primary")
-                
-        output = gr.Markdown(label="Generated Questions & Solutions")
-        
-        # Handle chapter selection change
-        chapter_dropdown.change(
-            fn=on_chapter_change,
-            inputs=[chapter_dropdown],
-            outputs=[subchapter_dropdown]
-        )
-        
-        # Handle generate button click
-        generate_button.click(
-            fn=generate_questions,
-            inputs=[
-                gr.State(lambda: chapters.index(chapter_dropdown.value) if chapter_dropdown.value in chapters else 0),
-                gr.State(lambda: calculus_curriculum[chapters.index(chapter_dropdown.value) if chapter_dropdown.value in chapters else 0]["subchapters"].index(subchapter_dropdown.value) if subchapter_dropdown.value in calculus_curriculum[chapters.index(chapter_dropdown.value) if chapter_dropdown.value in chapters else 0]["subchapters"] else 0),
-                difficulty,
-                num_questions
-            ],
-            outputs=[output]
-        )
-        
-        gr.Markdown("---")
-        gr.Markdown("Created by Kamagelo Mosia | Based on James Stewart's Calculus curriculum")
+    generator = CalculusTextbookGenerator()
+    generator.process_in_batches()
     
-    return demo
+    end_time = datetime.now()
+    duration = end_time - start_time
+    logging.info(f"Textbook generation completed in {duration}")
+    logging.info(f"Final textbook saved to {os.path.join(OUTPUT_DIR, 'textbook_latest.json')}")
 
-# Create and launch the interface
-demo = create_interface()
-demo.launch()
+if __name__ == "__main__":
+    main()