diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md new file mode 100644 index 0000000000000000000000000000000000000000..39c85e4f5b7406bc2f5a7a54b819b6ca8610d040 --- /dev/null +++ b/.agents/AGENTS.md @@ -0,0 +1,84 @@ +# Multiverse AI Studio — Agent Rules + +## User Learning Style + +This project is a **learning-first** environment. The user is building AND learning simultaneously. +These rules apply to every interaction in this workspace. + +--- + +### Rule 1 — Always Write Explanatory Comments + +Every piece of code written must include comments that explain: +- **What** the code does (the obvious) +- **Why** it does it that way (the reasoning) +- **How** it fits into the larger pipeline (the context) + +Do not write bare code. If a newcomer cannot understand the code from the comments alone, add more. + +Example style: +```python +# We use ThreadPoolExecutor here because model inference is CPU/GPU-bound +# (blocking), not I/O-bound. asyncio alone can't parallelize blocking work — +# it needs a thread pool to offload heavy computation without blocking the +# FastAPI event loop. +executor = ThreadPoolExecutor(max_workers=2) +``` + +--- + +### Rule 2 — Pause and Explain at Every Phase Step + +When implementing any phase step: +1. **Before coding** — briefly explain what is about to be built and why it exists in the pipeline. +2. **After coding** — explain what was just built, point to the key lines, and explain any non-obvious decisions. +3. **Invite doubts** — always end with an open invitation: *"Any questions before we move on?"* or similar. + +Do not chain multiple phase steps together without pausing. + +--- + +### Rule 3 — Take and Resolve Doubts Before Proceeding + +If the user asks a question mid-phase: +- Stop what you are doing. +- Answer the question fully. +- Check if the answer raised new questions. +- Only resume coding once the user signals they are ready. + +Never skip past a doubt to "keep momentum." + +--- + +### Rule 4 — User Workflow (READ THIS CAREFULLY) + +The user's development workflow is: + +``` +AI Studio (Google) + ↓ generates initial code scaffold +TRAE + ↓ organizes, structures, and assembles the codebase +Antigravity (this agent) + ↓ reviews, compares against implementation plan, checks quality +``` + +**Antigravity's role in this project is a REVIEWER and TEACHER — not the primary code generator.** + +When the user brings code for review: +- Compare it against the implementation plan phase by phase. +- Check that the model wrapper interface (`initialize/generate/cleanup`) is respected. +- Check that Stop & Review gate criteria are met. +- Explain what is correct, what could be improved, and why. +- Never silently fix things — always explain what was wrong and what the fix does. + +--- + +### Rule 5 — Explain Tradeoffs, Not Just Solutions + +Whenever a technical decision is made (or reviewed), explain: +- What alternatives existed +- Why this choice was made +- What the tradeoff is + +This prepares the user to defend decisions in interviews and discussions. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..080986d1d670d8d2669b671b4b62fa9039a51223 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# Generated AI outputs - these are large, binary, and regenerated on demand +generated_assets/ + +# Python cache files - automatically generated, not source code +__pycache__/ +*.pyc + +# Environment secrets - NEVER commit API keys or tokens to git +.env + +# Node.js dependencies - large directory, installed via npm install +node_modules/ + +# Next.js build artifacts - generated automatically on build +.next/ + +# macOS system files - OS-specific, not part of the project +.DS_Store + +# Python package metadata - generated during packaging +*.egg-info/ +test_apple.png diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..e2047a6ecbceb357f72dfa0d0fa4e6d2f66235fe --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# Stage 1: Build the React frontend +FROM node:18-alpine AS frontend-builder +WORKDIR /app/frontend +COPY frontend/package*.json ./ +RUN npm install +COPY frontend/ ./ +RUN npm run build + +# Stage 2: Serve using Python FastAPI +FROM python:3.10-slim +WORKDIR /app + +# Install system dependencies +# - git: needed for certain pip dependencies +# - libsndfile1: needed for SciPy audio file writes +# - ffmpeg: needed for imageio video compilations +RUN apt-get update && apt-get install -y \ + git \ + libsndfile1 \ + ffmpeg \ + && rm -rf /var/lib/apt/lists/* + +# Copy backend requirements first to leverage Docker build cache +COPY backend/requirements.txt ./backend/requirements.txt +RUN pip install --no-cache-dir -r backend/requirements.txt + +# Copy source code and frontend build output +COPY backend ./backend +COPY --from=frontend-builder /app/frontend/dist ./frontend/dist + +# Expose port 7860 (Hugging Face Spaces default container port) +EXPOSE 7860 + +# Start uvicorn server on port 7860, binding to all interfaces +CMD ["python", "-m", "uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "7860"] diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0bc0af8b5ec193e261601f4db707afd8cf73767d --- /dev/null +++ b/README.md @@ -0,0 +1,148 @@ +# Multiverse AI Studio + +A portfolio-grade, full-stack generative AI application that chains five Hugging Face models into a unified, coherent multimedia pipeline. A single human prompt is expanded into specialized instructions to generate a **base visual scene**, estimate its **3D depth geometry**, synthesize an **ambient background soundscape**, and render a **cinematic video** anchoring the visual track. + +Designed with a strict focus on system architecture, event-loop safety, memory management, and smooth user experience. + +--- + +## 🏗️ System Architecture + +### 1. Model Pipeline Flow +Every stage in the pipeline consumes something meaningful from the previous step. The visual assets (depth and video) are conditioned on the generated image, ensuring visual cohesion. + +```mermaid +graph TD + A[User Prompt] -->|POST /api/generate| B(Prompt Expansion LLM) + B -->|image_prompt| C(Image Generation Service) + B -->|audio_prompt| D(Audio Generation Service) + C -->|RGB Image| E(Depth Estimation Service) + C -->|RGB Image| F(Video Generation Service) + E -->|Depth Map| F + D -->|Audio Track| G[Final Coherent Scene] + F -->|MP4 Stream| G + + style C fill:#4f46e5,stroke:#333,stroke-width:2px,color:#fff + style E fill:#ec4899,stroke:#333,stroke-width:2px,color:#fff + style F fill:#db2777,stroke:#333,stroke-width:2px,color:#fff +``` + +### 2. Event-Loop & Threading Execution Flow +ML model inference is CPU/GPU-bound and blocking. To prevent blocking FastAPI's main asynchronous event loop, all inferences are offloaded to a background thread pool. + +``` +Browser FastAPI Route Background Worker Job Store + │ │ │ │ + │── POST /generate ──────>│ │ │ + │ (prompt payload) │── Create Job (UUID) ───────────────────────────────>│ (QUEUED) + │ │── Schedule run_pipeline ──>│ │ + │<── Return job_id ───────│ │ │ + │ │ │── Update Stage ────────>│ (EXPANDING...) + │ │ │── Run Prompt LLM │ + │ │ │── Run Image Gen │ + │ │ │── Update Asset (img) ──>│ (Image URL) + │ │ │── Run Depth Est │ + │ │ │── Update Asset (depth) ─>│ (Depth URL) + │ │ │── Run Audio/Video │ + │ │ │── Update final state ──>│ (COMPLETED) + │ │ │ │ + │── GET /result/{id} ────>│────────────────────────────────────────────────────>│ + │<── Returns assets ──────│<────────────────────────────────────────────────────│ +``` + +--- + +## ⚡ Key Features + +* **Progressive Polling & Rendering**: The frontend polls `/api/result/{job_id}`. Completed assets (like the base image) are rendered on the screen *immediately* while downstream stages are still processing. +* **Granular Memory/VRAM Management**: Chaining 5 heavy models sequentially can cause VRAM Out-of-Memory (OOM) crashes. Each model wrapper implements a strict `cleanup()` method that deletes pipeline instances, runs garbage collection (`gc.collect()`), and flushes PyTorch's CUDA memory cache (`torch.cuda.empty_cache()`) before loading the next stage. +* **Stage Error Isolation**: Wrap-around try/except boundaries guarantee that a single failed stage (e.g., depth map or audio timeout) does not crash the entire pipeline. The server flags a `PARTIAL_FAILURE` and delivers all other successfully compiled assets. +* **Interactive CSS Depth Slider**: Features a custom swipable comparison slider built using CSS `clip-path` polygon slicing for smooth 60fps comparisons between the visual base image and its calculated depth map. +* **Custom Media Players**: Custom glassmorphic React components for audio and video playback, including a dynamic pulsing audio waveform visualizer. + +--- + +## ⚙️ Project Setup + +### Prerequisites +* Python 3.10+ +* Node.js 18+ +* Hugging Face Access Token (for gated model downloads and Inference API) + +### 1. Environment Configuration +Create a `.env` file in the project root: +```env +HF_TOKEN=your_huggingface_access_token_here +MOCK_INFERENCE=False +FORCE_CPU_INFERENCE=False +``` + +#### Environment Variables Explained: +* `HF_TOKEN`: Your Hugging Face user access token (required for querying the cloud image generation API and downloading gated local models). +* `MOCK_INFERENCE`: + * `True` (Default DX): Bypasses all local and cloud model execution, returning mock visual, audio, and video assets in 1 second. Useful for testing UI components on any computer. + * `False` (Hybrid Production): Connects to the cloud and local machine learning models for real generation. +* `FORCE_CPU_INFERENCE`: + * `False` (Safe Fallback): If running `MOCK_INFERENCE=False` on a **CPU-only machine**, the backend will run cloud image generation and local depth maps, but will automatically bypass the heavy `MusicGen` and `i2vgen-xl` local models to prevent RAM exhaustion. + * `True` (Force CPU): Forces the backend to download, load, and execute the full MusicGen and Video models locally on your CPU. *Warning: MusicGen takes 2–5 minutes, and i2vgen-xl takes 20–45 minutes on CPU.* + +--- + +## 🚀 GPU & Production Execution Setup +If you want to run the **complete, real local PyTorch pipelines** (audio and video) on a GPU-enabled developer machine: + +1. **Install CUDA-enabled PyTorch**: Ensure your virtual environment is using a GPU-compiled version of PyTorch: + ```bash + pip install torch --index-url https://download.pytorch.org/whl/cu121 + ``` +2. **Configure environment**: Open your `.env` file and set: + ```env + HF_TOKEN=your_real_huggingface_token + MOCK_INFERENCE=False + FORCE_CPU_INFERENCE=False + ``` +3. **Launch**: Run uvicorn. The backend will automatically detect the GPU (`cuda`), log the status, and run the real local Hugging Face and Diffusers pipelines at maximum speed. + +### 2. Backend Installation +```bash +# Navigate to the backend directory +cd backend + +# Create a virtual environment +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Install dependencies +pip install -r requirements.txt + +# Start the development server +python -m uvicorn main:app --host 127.0.0.1 --port 8000 --reload +``` +The backend health check is available at `http://127.0.0.1:8000/api/health`. + +### 3. Frontend Installation +```bash +# Navigate to the frontend directory +cd frontend + +# Install packages +npm install + +# Start the Vite React development server +npm run dev +``` +Open `http://localhost:3000` to access the Multiverse AI Studio interface. + +--- + +## 📈 Engineering Decisions & Tradeoffs + +For a detailed analysis of our engineering decisions (such as choosing client polling over WebSockets, utilising an in-memory job store instead of Celery/Redis, and enforcing the BaseModel wrapper abstraction), please refer to the dedicated **[Tradeoffs and Decisions Report](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\docs\decisions\tradeoffs.md)**. + +--- + +## 🚀 Future Roadmap + +* **Server-Sent Events (SSE)**: Migrate the progressive rendering polling system to standard Server-Sent Events to push updates in real-time without client request overhead. +* **Persistent Database storage**: Replace the volatile in-memory dictionary with SQLite or PostgreSQL to keep user history across restarts. +* **Muxed Video Audio**: Integrate system FFmpeg binaries to merge (mux) the Stage 4 ambient soundscape directly into the Stage 5 MP4 video container. diff --git a/REPORT.md b/REPORT.md new file mode 100644 index 0000000000000000000000000000000000000000..44a0746a18d916e048e47f70d5f653f80b246108 --- /dev/null +++ b/REPORT.md @@ -0,0 +1,116 @@ +# Multiverse AI Studio - Development Report + +## Overview +This report summarizes all the work done on the Multiverse AI Studio project so far. + +--- + +## Table of Contents +1. [Backend Improvements](#backend-improvements) +2. [Frontend Updates](#frontend-updates) +3. [Files Modified](#files-modified) +4. [New Files Created](#new-files-created) + +--- + +## Backend Improvements + +### 1. Fixed Module Import Issues +All Python files in the backend were updated to use relative imports to avoid `ModuleNotFoundError` when running the server. +- Example changes: + - `from models.base import BaseModel` → `from .base import BaseModel` + - `from config import HF_TOKEN` → `from ..config import HF_TOKEN` + +### 2. Added Error Handling to Model Methods +Every model wrapper's `generate()` and `initialize()` methods now include try/except blocks to gracefully handle failures (like Out-of-Memory errors, API failures, etc.). +- Models updated: + - `PromptExpander` (already had good error handling) + - `ImageGenerator` + - `DepthEstimator` + - `AudioGenerator` + - `VideoGenerator` +- `execute_model_sync` in `pipeline.py` now also handles initialization errors + +### 3. Added Scene Description Support +- Added `set_job_scene_description` to `job_store.py` +- Updated `pipeline.py` to save the expanded scene description +- Updated `/api/result/{job_id}` in `routes.py` to return `scene_description` +- Updated backend `config.py` (no changes needed, but it's there) +- Added `scene_description` to `JobResult` in frontend API client + +### 4. Improved Job Store +- Added `scene_description` field to job state +- Added `error_at` timestamp for better error tracking + +### 5. Backend Startup Fixes +- `main.py` now loads dotenv before importing config +- `main.py` creates `OUTPUT_DIR` before mounting static files to avoid startup crashes + +--- + +## Frontend Updates + +### 1. Centralized API Client +Created `frontend/src/lib/api.ts` to handle all API calls with: +- Proper base URL (`http://localhost:8000/api`) +- Type definitions for API responses +- Asset URL fixing (prepends backend base URL to relative paths) + +### 2. Updated Pages +- **Home.tsx**: Now uses `generateAssets` from API client instead of direct fetch +- **Studio.tsx**: + - Now uses `getJobStatus` and `getJobResult` from API client + - Added collapsible panel to show the scene description + - Added error state for pipeline stages + - Added red error indicator and error message display + +### 3. Added Types +- `JobStatus` interface +- `JobResult` interface (includes `scene_description`) + +--- + +## Files Modified + +### Backend +- [`backend/main.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\main.py) +- [`backend/config.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\config.py) +- [`backend/api/routes.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\api\routes.py) +- [`backend/services/pipeline.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\services\pipeline.py) +- [`backend/utils/job_store.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\utils\job_store.py) +- [`backend/utils/file_manager.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\utils\file_manager.py) +- [`backend/models/prompt_expander.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\models\prompt_expander.py) +- [`backend/models/image_generator.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\models\image_generator.py) +- [`backend/models/depth_estimator.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\models\depth_estimator.py) +- [`backend/models/audio_generator.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\models\audio_generator.py) +- [`backend/models/video_generator.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\models\video_generator.py) + +### Frontend +- [`frontend/src/pages/Home.tsx`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\frontend\src\pages\Home.tsx) +- [`frontend/src/pages/Studio.tsx`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\frontend\src\pages\Studio.tsx) + +--- + +## New Files Created + +### Backend +- [`backend/api/__init__.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\api\__init__.py) (empty) +- [`backend/services/__init__.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\services\__init__.py) (empty) +- [`backend/utils/__init__.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\utils\__init__.py) (empty) +- [`backend/models/__init__.py`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\backend\models\__init__.py) (empty) + +### Frontend +- [`frontend/src/lib/api.ts`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\frontend\src\lib\api.ts) + +### Project Root +- [`.gitignore`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\.gitignore) +- [`TRAE-Skills/`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\TRAE-Skills) (cloned repo) +- [`REPORT.md`](file:///c:\AI Native founder\AI_Engineering\Projects\Multiverse_AI_Studio\REPORT.md) (this file!) + +--- + +## Next Steps +- Test the full pipeline end-to-end +- Add model download caching +- Add more comprehensive error handling in frontend +- Deploy to production diff --git a/TRAE-Skills/LICENSE b/TRAE-Skills/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..cb930b96d2a2b6d9bf5bcb11217ee9ea45af7755 --- /dev/null +++ b/TRAE-Skills/LICENSE @@ -0,0 +1,64 @@ +TRAE Skills License Agreement +Version 1.0 + +Copyright (c) 2026 HighMark-IT + +1. GRANT OF RIGHTS + +You are granted a non-exclusive, royalty-free right to use, copy, modify, and distribute this software and its associated documentation files (the "Software") for personal, educational, and commercial purposes, subject to the conditions outlined in this License. + +2. CONDITIONS AND RESTRICTIONS + +A. Proper Attribution Required + - You MUST include clear attribution to this repository (https://github.com/HighMark-31/TRAE-Agents) and its author (HighMark-IT) in any derivative work, product, or project that incorporates or uses the Software or its agent structures. + - Attribution must be visible and accessible to end users. + +B. No Republication of Original Agents + - You MAY NOT republish, redistribute, or present the original agents or substantially unmodified versions of them as your own public collection, product suite, or template library. + - You cannot claim ownership of the "TRAE Agents" collection, the specific agent names, or the overall ecosystem as originally created in this repository. + - Minor modifications (e.g., parameter tweaks, prompt rewording) do not constitute a new original work and remain subject to this restriction. + +C. Prohibited Agent Rebranding + - You MAY NOT use the names "TRAE Agents", "TRAE-Agents", or the specific agent names (e.g., "General Coordinator", "Code Optimizer", "Security Sentinel") as if they were your original creation or branding. + - You must create distinctly different naming and branding if you publish agent collections based on this work. + +D. Original Works Permitted + - Creating your own agents or agent frameworks INSPIRED BY this project is permitted, provided they are: + 1. Materially different in structure, purpose, or implementation + 2. Given original names and branding + 3. Properly attributed to this repository as inspiration + +3. COMMERCIAL USE + +You are permitted to use the Software in commercial products and services, provided that: + - You maintain proper attribution to this repository + - You do not republish the original agent collection as your own + - You do not claim ownership of the TRAE Agents ecosystem + +4. MODIFICATIONS AND DERIVATIVES + +You may modify the Software for your own use. If you distribute modified versions, you must: + - Clearly mark modifications as your own + - Maintain attribution to the original work + - Not present near-identical versions as original creations + +5. NO WARRANTY + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +6. ENFORCEMENT + +Breach of conditions outlined in Section 2 may result in: + - Cease and desist notices + - Requests for content removal from public platforms + - Legal action to enforce compliance with this License + +7. SEVERABILITY + +If any provision of this License is found to be unenforceable, the remaining provisions shall remain in full force and effect. + +8. GOVERNING LAW + +This License is governed by applicable copyright and intellectual property laws. The Copyright holder reserves the right to modify these terms with notice. + +By using this Software, you acknowledge that you have read, understood, and agree to be bound by all terms of this License. diff --git a/TRAE-Skills/README.md b/TRAE-Skills/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e570110c968773dca161a75baf711d64e487f99d --- /dev/null +++ b/TRAE-Skills/README.md @@ -0,0 +1,240 @@ +
+ +### ✨ The Ultimate AI Skill Library for Modern Development ✨ + +
+ +# [TRAE](https://www.trae.ai/) Skills Collection + +**_The Largest, Most Complete, and Popular Collection of TRAE Skills._** + +A powerful collection of **expert-level TRAE Skills**, designed to standardize and elevate every stage of modern software development. +These skills act as a knowledge base for agents, enabling them to generate production-ready code, follow best practices, and handle complex architectural tasks across multiple stacks. + +Perfect for developers, engineers, and teams seeking **consistency, quality, and expert-level output** in every interaction. + + +--- + +[![Star the project](https://img.shields.io/badge/⭐_Star_the_Project-black?style=for-the-badge)](https://github.com/HighMark-31/TRAE-Skills/stargazers) ![Visitors](https://visitor-badge.laobi.icu/badge?page_id=HighMark-31.TRAE-Skills) + +--- + +## 📍 Skill List + +| Category | Skill Name | Info | View | +| :--- | :--- | :--- | :--- | +| **🔹 ️ Frontend** | Accessibility Audit | [ℹ️](./frontend/Accessibility_Audit.md) | [👆 View](./frontend/Accessibility_Audit.md) | +| | API Data Fetching (TanStack) | [ℹ️](./frontend/API_Data_Fetching_TanStack.md) | [👆 View](./frontend/API_Data_Fetching_TanStack.md) | +| | Authentication Strategy Selection | [ℹ️](./architecture/Authentication_Strategy_Selection.md) | [👆 View](./architecture/Authentication_Strategy_Selection.md) | +| | BFF Pattern Implementation | [ℹ️](./architecture/BFF_Pattern_Implementation.md) | [👆 View](./architecture/BFF_Pattern_Implementation.md) | +| | Code Review Security Checklist | [ℹ️](./security/Code_Review_Security.md) | [👆 View](./security/Code_Review_Security.md) | +| | Custom React Hook Creation | [ℹ️](./frontend/Custom_React_Hook_Creation.md) | [👆 View](./frontend/Custom_React_Hook_Creation.md) | +| | Database Selection (SQL vs NoSQL) | [ℹ️](./architecture/Database_Selection_SQL_vs_NoSQL.md) | [👆 View](./architecture/Database_Selection_SQL_vs_NoSQL.md) | +| | E2E Testing (Playwright) | [ℹ️](./testing/E2E_Testing_Playwright.md) | [👆 View](./testing/E2E_Testing_Playwright.md) | +| | Event Driven Architecture | [ℹ️](./architecture/Event_Driven_Architecture_Basics.md) | [👆 View](./architecture/Event_Driven_Architecture_Basics.md) | +| | Form Handling (ReactHookForm) | [ℹ️](./frontend/Form_Handling_ReactHookForm.md) | [👆 View](./frontend/Form_Handling_ReactHookForm.md) | +| | Frontend Error Boundary | [ℹ️](./frontend/Frontend_Error_Boundary.md) | [👆 View](./frontend/Frontend_Error_Boundary.md) | +| | Frontend-Backend Communication | [ℹ️](./architecture/Frontend_Backend_Communication_Patterns.md) | [👆 View](./architecture/Frontend_Backend_Communication_Patterns.md) | +| | Global State (Redux Toolkit) | [ℹ️](./frontend/Global_State_Management_Redux.md) | [👆 View](./frontend/Global_State_Management_Redux.md) | +| | Input Validation (Zod) | [ℹ️](./security/Input_Validation_Zod.md) | [👆 View](./security/Input_Validation_Zod.md) | +| | Internationalization (i18n) | [ℹ️](./frontend/Internationalization_i18n.md) | [👆 View](./frontend/Internationalization_i18n.md) | +| | JWT Authentication | [ℹ️](./security/JWT_Authentication.md) | [👆 View](./security/JWT_Authentication.md) | +| | Microservices vs Monolith | [ℹ️](./architecture/Microservices_vs_Monolith_Decision.md) | [👆 View](./architecture/Microservices_vs_Monolith_Decision.md) | +| | Multi-Tenancy Architecture | [ℹ️](./architecture/Multi_Tenancy_Architecture.md) | [👆 View](./architecture/Multi_Tenancy_Architecture.md) | +| | Rate Limiting (Redis) | [ℹ️](./security/Rate_Limiting_Redis.md) | [👆 View](./security/Rate_Limiting_Redis.md) | +| | RBAC Implementation | [ℹ️](./security/RBAC_Implementation.md) | [👆 View](./security/RBAC_Implementation.md) | +| | React Component Optimization | [ℹ️](./frontend/React_Component_Optimization.md) | [👆 View](./frontend/React_Component_Optimization.md) | +| | Responsive UI (Tailwind) | [ℹ️](./frontend/Responsive_UI_Design_Tailwind.md) | [👆 View](./frontend/Responsive_UI_Design_Tailwind.md) | +| | REST vs GraphQL Selection | [ℹ️](./architecture/REST_vs_GraphQL_Selection.md) | [👆 View](./architecture/REST_vs_GraphQL_Selection.md) | +| | Route Protection (React Router) | [ℹ️](./frontend/Route_Protection_React_Router.md) | [👆 View](./frontend/Route_Protection_React_Router.md) | +| | Scaling Strategies | [ℹ️](./architecture/Scaling_Strategies_Horizontal_vs_Vertical.md) | [👆 View](./architecture/Scaling_Strategies_Horizontal_vs_Vertical.md) | +| | Secure Env Var Handling | [ℹ️](./security/Secure_Env_Var_Handling.md) | [👆 View](./security/Secure_Env_Var_Handling.md) | +| | Serverless Architecture | [ℹ️](./architecture/Serverless_Architecture_Considerations.md) | [👆 View](./architecture/Serverless_Architecture_Considerations.md) | +| | SSR vs CSR Decision | [ℹ️](./architecture/SSR_vs_CSR_Decision_Matrix.md) | [👆 View](./architecture/SSR_vs_CSR_Decision_Matrix.md) | +| | Stack Selection Criteria | [ℹ️](./architecture/Stack_Selection_Criteria.md) | [👆 View](./architecture/Stack_Selection_Criteria.md) | +| | Unit Test Generation (Jest) | [ℹ️](./testing/Unit_Test_Generation_Jest.md) | [👆 View](./testing/Unit_Test_Generation_Jest.md) | +| **💻 Backend** | API REST Endpoint Design | [ℹ️](./backend/API_REST_Endpoint_Design.md) | [👆 View](./backend/API_REST_Endpoint_Design.md) | +| | API Versioning Strategies | [ℹ️](./backend/API_Versioning_Strategies.md) | [👆 View](./backend/API_Versioning_Strategies.md) | +| | Background Jobs with BullMQ | [ℹ️](./backend/Background_Jobs_BullMQ.md) | [👆 View](./backend/Background_Jobs_BullMQ.md) | +| | Caching Strategy (Redis) | [ℹ️](./backend/Caching_Strategy_Redis.md) | [👆 View](./backend/Caching_Strategy_Redis.md) | +| | Database Locking Strategies | [ℹ️](./backend/Database_Locking_Strategies.md) | [👆 View](./backend/Database_Locking_Strategies.md) | +| | Database Schema Migration | [ℹ️](./backend/Database_Schema_Migration.md) | [👆 View](./backend/Database_Schema_Migration.md) | +| | Database Seeding | [ℹ️](./backend/Database_Seeding.md) | [👆 View](./backend/Database_Seeding.md) | +| | Error Handling (Express) | [ℹ️](./backend/Error_Handling_Express.md) | [👆 View](./backend/Error_Handling_Express.md) | +| | Express Middleware Creation | [ℹ️](./backend/Express_Middleware_Creation.md) | [👆 View](./backend/Express_Middleware_Creation.md) | +| | Feature Flag Implementation | [ℹ️](./backend/Feature_Flag_Implementation.md) | [👆 View](./backend/Feature_Flag_Implementation.md) | +| | File Storage (S3/MinIO) | [ℹ️](./backend/File_Storage_S3_MinIO.md) | [👆 View](./backend/File_Storage_S3_MinIO.md) | +| | File Upload Handling | [ℹ️](./backend/File_Upload_Handling.md) | [👆 View](./backend/File_Upload_Handling.md) | +| | GraphQL Schema Design | [ℹ️](./backend/GraphQL_Schema_Design.md) | [👆 View](./backend/GraphQL_Schema_Design.md) | +| | gRPC Service Implementation | [ℹ️](./backend/gRPC_Service_Implementation.md) | [👆 View](./backend/gRPC_Service_Implementation.md) | +| | Handling Distributed Transactions (Sagas) | [ℹ️](./backend/Distributed_Transactions_Sagas.md) | [👆 View](./backend/Distributed_Transactions_Sagas.md) | +| | Health Check Endpoint | [ℹ️](./backend/Health_Check_Endpoint.md) | [👆 View](./backend/Health_Check_Endpoint.md) | +| | Implementing OAuth2 Providers | [ℹ️](./backend/OAuth2_Provider_Implementation.md) | [👆 View](./backend/OAuth2_Provider_Implementation.md) | +| | Implementing Search (Elasticsearch) | [ℹ️](./backend/Elasticsearch_Integration.md) | [👆 View](./backend/Elasticsearch_Integration.md) | +| | Logger Configuration (Winston) | [ℹ️](./backend/Logger_Configuration_Winston.md) | [👆 View](./backend/Logger_Configuration_Winston.md) | +| | Message Queue Implementation | [ℹ️](./backend/Message_Queue_Implementation.md) | [👆 View](./backend/Message_Queue_Implementation.md) | +| | Microservice Communication | [ℹ️](./backend/Microservice_Communication.md) | [👆 View](./backend/Microservice_Communication.md) | +| | MongoDB Aggregation Pipeline | [ℹ️](./backend/MongoDB_Aggregation.md) | [👆 View](./backend/MongoDB_Aggregation.md) | +| | Multi-factor Authentication (MFA) | [ℹ️](./backend/MFA_Implementation.md) | [👆 View](./backend/MFA_Implementation.md) | +| | Node.js Stream Processing | [ℹ️](./backend/Nodejs_Streams.md) | [👆 View](./backend/Nodejs_Streams.md) | +| | Pagination Implementation | [ℹ️](./backend/Pagination_Implementation.md) | [👆 View](./backend/Pagination_Implementation.md) | +| | PostgreSQL Indexing Strategies | [ℹ️](./backend/PostgreSQL_Indexing.md) | [👆 View](./backend/PostgreSQL_Indexing.md) | +| | Prisma Schema Design | [ℹ️](./backend/Prisma_Schema_Design.md) | [👆 View](./backend/Prisma_Schema_Design.md) | +| | Redis Data Structures | [ℹ️](./backend/Redis_Data_Structures.md) | [👆 View](./backend/Redis_Data_Structures.md) | +| | Refactoring Legacy Controller | [ℹ️](./backend/Refactoring_Legacy_Controller.md) | [👆 View](./backend/Refactoring_Legacy_Controller.md) | +| | Role-Based Access Control (Advanced) | [ℹ️](./backend/Advanced_RBAC.md) | [👆 View](./backend/Advanced_RBAC.md) | +| | Session Management Best Practices | [ℹ️](./backend/Session_Management_Best_Practices.md) | [👆 View](./backend/Session_Management_Best_Practices.md) | +| | Soft Delete Implementation | [ℹ️](./backend/Soft_Delete_Implementation.md) | [👆 View](./backend/Soft_Delete_Implementation.md) | +| | SQL Query Optimization | [ℹ️](./backend/SQL_Query_Optimization.md) | [👆 View](./backend/SQL_Query_Optimization.md) | +| | Swagger Documentation | [ℹ️](./backend/Swagger_Documentation.md) | [👆 View](./backend/Swagger_Documentation.md) | +| | Transaction Management | [ℹ️](./backend/Transaction_Management.md) | [👆 View](./backend/Transaction_Management.md) | +| | TypeORM Entity Relations | [ℹ️](./backend/TypeORM_Relations.md) | [👆 View](./backend/TypeORM_Relations.md) | +| | Webhooks Implementation | [ℹ️](./backend/Webhooks_Implementation.md) | [👆 View](./backend/Webhooks_Implementation.md) | +| | WebSocket Implementation | [ℹ️](./backend/WebSocket_Implementation.md) | [👆 View](./backend/WebSocket_Implementation.md) | +| | Real-time Data Processing (Kafka) | [ℹ️](./backend/Real-time_Data_Processing_Kafka.md) | [👆 View](./backend/Real-time_Data_Processing_Kafka.md) | +| | Real-time GraphQL Subscriptions | [ℹ️](./backend/GraphQL_Subscriptions_Realtime.md) | [👆 View](./backend/GraphQL_Subscriptions_Realtime.md) | +| | WebSocket Scalability (Socket.io) | [ℹ️](./backend/WebSocket_Scalability_SocketIO.md) | [👆 View](./backend/WebSocket_Scalability_SocketIO.md) | +| | Type-Safe APIs with tRPC | [ℹ️](./backend/Type_Safe_APIs_tRPC.md) | [👆 View](./backend/Type_Safe_APIs_tRPC.md) | +| | Serverless Streaming & Webhooks at Scale | [ℹ️](./backend/Serverless_Streaming_Webhooks.md) | [👆 View](./backend/Serverless_Streaming_Webhooks.md) | +| **🖥️ Frontend** | Browser Storage (LocalStorage/IndexedDB) | [ℹ️](./frontend/Browser_Storage.md) | [👆 View](./frontend/Browser_Storage.md) | +| | Canvas/WebGL Basics (Three.js) | [ℹ️](./frontend/Canvas_Threejs_Basics.md) | [👆 View](./frontend/Canvas_Threejs_Basics.md) | +| | CSS Grid vs Flexbox Guide | [ℹ️](./frontend/CSS_Grid_vs_Flexbox.md) | [👆 View](./frontend/CSS_Grid_vs_Flexbox.md) | +| | Dark Mode Implementation | [ℹ️](./frontend/Dark_Mode_Implementation.md) | [👆 View](./frontend/Dark_Mode_Implementation.md) | +| | Handling Large Lists (Virtualization) | [ℹ️](./frontend/Handling_Large_Lists_Virtualization.md) | [👆 View](./frontend/Handling_Large_Lists_Virtualization.md) | +| | Mobile-First Design Principles | [ℹ️](./frontend/Mobile_First_Design.md) | [👆 View](./frontend/Mobile_First_Design.md) | +| | Next.js App Router Migration | [ℹ️](./frontend/Nextjs_App_Router.md) | [👆 View](./frontend/Nextjs_App_Router.md) | +| | Optimizing Web Vitals | [ℹ️](./frontend/Web_Vitals_Optimization.md) | [👆 View](./frontend/Web_Vitals_Optimization.md) | +| | PWA Implementation | [ℹ️](./frontend/PWA_Implementation.md) | [👆 View](./frontend/PWA_Implementation.md) | +| | React Context vs Zustand | [ℹ️](./frontend/React_Context_vs_Zustand.md) | [👆 View](./frontend/React_Context_vs_Zustand.md) | +| | Storybook Component Documentation | [ℹ️](./frontend/Storybook_Component_Documentation.md) | [👆 View](./frontend/Storybook_Component_Documentation.md) | +| | SVG Animation Techniques | [ℹ️](./frontend/SVG_Animation_Techniques.md) | [👆 View](./frontend/SVG_Animation_Techniques.md) | +| | Testing React Components (RTL) | [ℹ️](./frontend/React_Testing_Library.md) | [👆 View](./frontend/React_Testing_Library.md) | +| | Web Workers for Heavy Computation | [ℹ️](./frontend/Web_Workers.md) | [👆 View](./frontend/Web_Workers.md) | +| | Advanced WebGL with Three.js | [ℹ️](./frontend/WebGL_Advanced_Threejs.md) | [👆 View](./frontend/WebGL_Advanced_Threejs.md) | +| | Advanced Web Animations with Framer Motion | [ℹ️](./frontend/Web_Animations_Framer_Motion.md) | [👆 View](./frontend/Web_Animations_Framer_Motion.md) | +| **📱 Mobile** | App Store Deployment Guide | [ℹ️](./mobile/App_Store_Deployment_Guide.md) | [👆 View](./mobile/App_Store_Deployment_Guide.md) | +| | Mobile Device Features | [ℹ️](./mobile/Mobile_Device_Features.md) | [👆 View](./mobile/Mobile_Device_Features.md) | +| | Mobile UI Styling (NativeWind) | [ℹ️](./mobile/Mobile_UI_Styling_NativeWind.md) | [👆 View](./mobile/Mobile_UI_Styling_NativeWind.md) | +| | Offline-First Mobile Architecture | [ℹ️](./mobile/Offline_First_Mobile_Architecture.md) | [👆 View](./mobile/Offline_First_Mobile_Architecture.md) | +| | Push Notifications Setup | [ℹ️](./mobile/Push_Notifications_Setup.md) | [👆 View](./mobile/Push_Notifications_Setup.md) | +| | React Native Navigation | [ℹ️](./mobile/React_Native_Navigation.md) | [👆 View](./mobile/React_Native_Navigation.md) | +| | React Native Reanimated | [ℹ️](./mobile/React_Native_Reanimated.md) | [👆 View](./mobile/React_Native_Reanimated.md) | +| | React Native Setup (Expo) | [ℹ️](./mobile/React_Native_Setup_Expo.md) | [👆 View](./mobile/React_Native_Setup_Expo.md) | +| | Biometric Authentication (Expo) | [ℹ️](./mobile/Biometric_Authentication_Expo.md) | [👆 View](./mobile/Biometric_Authentication_Expo.md) | +| | Deep Linking (React Navigation) | [ℹ️](./mobile/Deep_Linking_React_Navigation.md) | [👆 View](./mobile/Deep_Linking_React_Navigation.md) | +| | Flutter Advanced State Management | [ℹ️](./mobile/Flutter_Advanced_State_Management.md) | [👆 View](./mobile/Flutter_Advanced_State_Management.md) | +| | React Native Performance Optimization | [ℹ️](./mobile/React_Native_Performance_Optimization.md) | [👆 View](./mobile/React_Native_Performance_Optimization.md) | +| **🔧 DevOps** | Ansible Playbook Creation | [ℹ️](./devops/Ansible_Playbook_Creation.md) | [👆 View](./devops/Ansible_Playbook_Creation.md) | +| | Automated Database Backups | [ℹ️](./devops/Automated_Database_Backups.md) | [👆 View](./devops/Automated_Database_Backups.md) | +| | AWS Lambda Function Design | [ℹ️](./devops/AWS_Lambda_Function_Design.md) | [👆 View](./devops/AWS_Lambda_Function_Design.md) | +| | AWS Secrets Manager Integration | [ℹ️](./devops/AWS_Secrets_Manager_Integration.md) | [👆 View](./devops/AWS_Secrets_Manager_Integration.md) | +| | Azure Functions Basics | [ℹ️](./devops/Azure_Functions_Basics.md) | [👆 View](./devops/Azure_Functions_Basics.md) | +| | Blue/Green Deployment | [ℹ️](./devops/Blue_Green_Deployment_Strategy.md) | [👆 View](./devops/Blue_Green_Deployment_Strategy.md) | +| | Chaos Engineering Basics | [ℹ️](./devops/Chaos_Engineering_Basics.md) | [👆 View](./devops/Chaos_Engineering_Basics.md) | +| | CI Pipeline (GitHub Actions) | [ℹ️](./devops/CI_Pipeline_GitHub_Actions.md) | [👆 View](./devops/CI_Pipeline_GitHub_Actions.md) | +| | Cost Optimization (AWS/Cloud) | [ℹ️](./devops/Cost_Optimization_Cloud.md) | [👆 View](./devops/Cost_Optimization_Cloud.md) | +| | Dependency Update Audit | [ℹ️](./devops/Dependency_Update_Audit.md) | [👆 View](./devops/Dependency_Update_Audit.md) | +| | Docker Containerization (Node) | [ℹ️](./devops/Docker_Containerization_Node.md) | [👆 View](./devops/Docker_Containerization_Node.md) | +| | Git Branching Strategy | [ℹ️](./devops/Git_Branching_Strategy.md) | [👆 View](./devops/Git_Branching_Strategy.md) | +| | Google Cloud Run Deployment | [ℹ️](./devops/Google_Cloud_Run_Deployment.md) | [👆 View](./devops/Google_Cloud_Run_Deployment.md) | +| | Infrastructure as Code (Terraform) | [ℹ️](./devops/Infrastructure_as_Code_Terraform.md) | [👆 View](./devops/Infrastructure_as_Code_Terraform.md) | +| | K8s Deployment Manifests | [ℹ️](./devops/Kubernetes_Deployment_Manifests.md) | [👆 View](./devops/Kubernetes_Deployment_Manifests.md) | +| | Log Aggregation (ELK) | [ℹ️](./devops/Log_Aggregation_ELK.md) | [👆 View](./devops/Log_Aggregation_ELK.md) | +| | Monitoring with Datadog | [ℹ️](./devops/Monitoring_with_Datadog.md) | [👆 View](./devops/Monitoring_with_Datadog.md) | +| | Nginx Reverse Proxy Setup | [ℹ️](./devops/Nginx_Reverse_Proxy_Setup.md) | [👆 View](./devops/Nginx_Reverse_Proxy_Setup.md) | +| | Prometheus & Grafana Monitoring | [ℹ️](./devops/Prometheus_Grafana_Monitoring.md) | [👆 View](./devops/Prometheus_Grafana_Monitoring.md) | +| | Sentry Error Tracking Setup | [ℹ️](./devops/Sentry_Error_Tracking.md) | [👆 View](./devops/Sentry_Error_Tracking.md) | +| | Serverless Framework Setup | [ℹ️](./devops/Serverless_Framework_Setup.md) | [👆 View](./devops/Serverless_Framework_Setup.md) | +| | SSL/TLS Setup (Certbot) | [ℹ️](./devops/SSL_TLS_Certbot_Setup.md) | [👆 View](./devops/SSL_TLS_Certbot_Setup.md) | +| | Terraform Best Practices (Advanced) | [ℹ️](./devops/Terraform_Advanced.md) | [👆 View](./devops/Terraform_Advanced.md) | +| | Docker Swarm Orchestration | [ℹ️](./devops/Docker_Swarm_Orchestration.md) | [👆 View](./devops/Docker_Swarm_Orchestration.md) | +| | Kubernetes Helm Charts | [ℹ️](./devops/Kubernetes_Helm_Charts.md) | [👆 View](./devops/Kubernetes_Helm_Charts.md) | +| | Edge Computing with Vercel & Cloudflare Workers | [ℹ️](./devops/Edge_Computing_Vercel_Cloudflare.md) | [👆 View](./devops/Edge_Computing_Vercel_Cloudflare.md) | +| **🛡️ Security** | Content Security Policy (CSP) Setup | [ℹ️](./security/Content_Security_Policy_CSP.md) | [👆 View](./security/Content_Security_Policy_CSP.md) | +| | CSRF Protection Strategies | [ℹ️](./security/CSRF_Protection_Strategies.md) | [👆 View](./security/CSRF_Protection_Strategies.md) | +| | Dependency Vulnerability Scanning | [ℹ️](./security/Dependency_Vulnerability_Scanning.md) | [👆 View](./security/Dependency_Vulnerability_Scanning.md) | +| | OWASP Top 10 Mitigation | [ℹ️](./security/OWASP_Top_10_Mitigation.md) | [👆 View](./security/OWASP_Top_10_Mitigation.md) | +| | Secret Scanning in CI/CD | [ℹ️](./security/Secret_Scanning_CI_CD.md) | [👆 View](./security/Secret_Scanning_CI_CD.md) | +| | XSS Prevention Guide | [ℹ️](./security/XSS_Prevention_Guide.md) | [👆 View](./security/XSS_Prevention_Guide.md) | +| | CORS Configuration Best Practices | [ℹ️](./security/CORS_Configuration_Best_Practices.md) | [👆 View](./security/CORS_Configuration_Best_Practices.md) | +| | SQL Injection Prevention | [ℹ️](./security/SQL_Injection_Prevention.md) | [👆 View](./security/SQL_Injection_Prevention.md) | +| | OAuth2 & OIDC Implementation | [ℹ️](./security/OAuth2_OIDC_Implementation.md) | [👆 View](./security/OAuth2_OIDC_Implementation.md) | +| | Password Hashing Best Practices | [ℹ️](./security/Password_Hashing_Best_Practices.md) | [👆 View](./security/Password_Hashing_Best_Practices.md) | +| | Multi-factor Authentication (MFA) | [ℹ️](./backend/MFA_Implementation.md) | [👆 View](./backend/MFA_Implementation.md) | +| | API Security Penetration Testing | [ℹ️](./security/API_Security_Penetration_Testing.md) | [👆 View](./security/API_Security_Penetration_Testing.md) | +| | Zero Trust Architecture Implementation | [ℹ️](./security/Zero_Trust_Architecture.md) | [👆 View](./security/Zero_Trust_Architecture.md) | +| **🧪 Testing** | API Integration Testing (Supertest) | [ℹ️](./testing/API_Integration_Testing_Supertest.md) | [👆 View](./testing/API_Integration_Testing_Supertest.md) | +| | Visual Regression Testing (Playwright) | [ℹ️](./testing/Visual_Regression_Testing_Playwright.md) | [👆 View](./testing/Visual_Regression_Testing_Playwright.md) | +| | Load & Performance Testing (k6) | [ℹ️](./testing/Load_Testing_k6.md) | [👆 View](./testing/Load_Testing_k6.md) | +| | Contract Testing (Pact) | [ℹ️](./testing/Consumer_Driven_Contract_Testing_Pact.md) | [👆 View](./testing/Consumer_Driven_Contract_Testing_Pact.md) | +| | Mutation Testing (Stryker) | [ℹ️](./testing/Mutation_Testing_Stryker.md) | [👆 View](./testing/Mutation_Testing_Stryker.md) | +| | Automated Accessibility Testing (Axe-core) | [ℹ️](./testing/Automated_Accessibility_Testing_Axe.md) | [👆 View](./testing/Automated_Accessibility_Testing_Axe.md) | +| | Snapshot Testing (Jest) | [ℹ️](./testing/Snapshot_Testing_Jest.md) | [👆 View](./testing/Snapshot_Testing_Jest.md) | +| | Test-Driven Development (TDD) | [ℹ️](./testing/Test_Driven_Development_TDD.md) | [👆 View](./testing/Test_Driven_Development_TDD.md) | +| | Component Testing (RTL) | [ℹ️](./testing/Component_Testing_React_Testing_Library.md) | [👆 View](./testing/Component_Testing_React_Testing_Library.md) | +| | Mocking External Services (Jest) | [ℹ️](./testing/Mocking_External_Services_Jest.md) | [👆 View](./testing/Mocking_External_Services_Jest.md) | +| | Test Coverage & Quality (Istanbul) | [ℹ️](./testing/Test_Coverage_Quality_Istanbul.md) | [👆 View](./testing/Test_Coverage_Quality_Istanbul.md) | +| | Performance Testing with Lighthouse | [ℹ️](./testing/Performance_Testing_Lighthouse.md) | [👆 View](./testing/Performance_Testing_Lighthouse.md) | +| | Advanced End-to-End Testing with Playwright | [ℹ️](./testing/End-to-End_Testing_Playwright_Advanced.md) | [👆 View](./testing/End-to-End_Testing_Playwright_Advanced.md) | +| **🤖 AI Engineering** | AI Agent Design Patterns | [ℹ️](./ai_engineering/AI_Agent_Design_Patterns.md) | [👆 View](./ai_engineering/AI_Agent_Design_Patterns.md) | +| | AI Model Evaluation | [ℹ️](./ai_engineering/AI_Model_Evaluation.md) | [👆 View](./ai_engineering/AI_Model_Evaluation.md) | +| | Fine-tuning Basics | [ℹ️](./ai_engineering/Fine_tuning_Basics.md) | [👆 View](./ai_engineering/Fine_tuning_Basics.md) | +| | LangChain Basics | [ℹ️](./ai_engineering/LangChain_Basics.md) | [👆 View](./ai_engineering/LangChain_Basics.md) | +| | Local LLM Running (Ollama) | [ℹ️](./ai_engineering/Local_LLM_Running_Ollama.md) | [👆 View](./ai_engineering/Local_LLM_Running_Ollama.md) | +| | OpenAI API Integration | [ℹ️](./ai_engineering/OpenAI_API_Integration.md) | [👆 View](./ai_engineering/OpenAI_API_Integration.md) | +| | Prompt Engineering Basics | [ℹ️](./ai_engineering/Prompt_Engineering_Basics.md) | [👆 View](./ai_engineering/Prompt_Engineering_Basics.md) | +| | RAG System Architecture | [ℹ️](./ai_engineering/RAG_System_Architecture.md) | [👆 View](./ai_engineering/RAG_System_Architecture.md) | +| | Speech-to-Text Implementation (Whisper) | [ℹ️](./ai_engineering/Speech_to_Text_Whisper.md) | [👆 View](./ai_engineering/Speech_to_Text_Whisper.md) | +| | Vector Database Setup | [ℹ️](./ai_engineering/Vector_Database_Setup.md) | [👆 View](./ai_engineering/Vector_Database_Setup.md) | +| | ML Model Quantization | [ℹ️](./ai_engineering/ML_Model_Quantization.md) | [👆 View](./ai_engineering/ML_Model_Quantization.md) | +| | Time Series Forecasting | [ℹ️](./ai_engineering/Time_Series_Forecasting.md) | [👆 View](./ai_engineering/Time_Series_Forecasting.md) | +| | Distributed Training (Horovod) | [ℹ️](./ai_engineering/Distributed_Training_Horovod.md) | [👆 View](./ai_engineering/Distributed_Training_Horovod.md) | +| | Computer Vision Object Detection | [ℹ️](./ai_engineering/Computer_Vision_Object_Detection.md) | [👆 View](./ai_engineering/Computer_Vision_Object_Detection.md) | +| | Data Drift Detection | [ℹ️](./ai_engineering/Data_Drift_Detection.md) | [👆 View](./ai_engineering/Data_Drift_Detection.md) | +| | Generative AI Image Synthesis | [ℹ️](./ai_engineering/Generative_AI_Image_Synthesis.md) | [👆 View](./ai_engineering/Generative_AI_Image_Synthesis.md) | +| | Natural Language to SQL (NL2SQL) | [ℹ️](./ai_engineering/Natural_Language_to_SQL.md) | [👆 View](./ai_engineering/Natural_Language_to_SQL.md) | +| | AI Agents with LangGraph | [ℹ️](./ai_engineering/AI_Agents_LangGraph.md) | [👆 View](./ai_engineering/AI_Agents_LangGraph.md) | +| | Advanced Vector Databases (Pinecone & Weaviate) | [ℹ️](./ai_engineering/Vector_Databases_Pinecone_Weaviate.md) | [👆 View](./ai_engineering/Vector_Databases_Pinecone_Weaviate.md) | +| | Time Series Forecasting with LSTM | [ℹ️](./ai_engineering/Time_Series_Forecasting_LSTM.md) | [👆 View](./ai_engineering/Time_Series_Forecasting_LSTM.md) | +| | Recommender Systems with Collaborative Filtering | [ℹ️](./ai_engineering/Recommender_Systems_Collaborative_Filtering.md) | [👆 View](./ai_engineering/Recommender_Systems_Collaborative_Filtering.md) | +| **🏗️ Architecture** | Adapter Pattern in TypeScript | [ℹ️](./architecture/Adapter_Pattern_TypeScript.md) | [👆 View](./architecture/Adapter_Pattern_TypeScript.md) | +| | Clean Architecture in Node.js | [ℹ️](./architecture/Clean_Architecture_Node.md) | [👆 View](./architecture/Clean_Architecture_Node.md) | +| | CQRS Pattern Implementation | [ℹ️](./architecture/CQRS_Pattern_Implementation.md) | [👆 View](./architecture/CQRS_Pattern_Implementation.md) | +| | Domain-Driven Design (DDD) Basics | [ℹ️](./architecture/Domain_Driven_Design_Basics.md) | [👆 View](./architecture/Domain_Driven_Design_Basics.md) | +| | CQRS Implementation | [ℹ️](./architecture/CQRS_Implementation.md) | [👆 View](./architecture/CQRS_Implementation.md) | +| | Event Sourcing Pattern | [ℹ️](./architecture/Event_Sourcing_Pattern.md) | [👆 View](./architecture/Event_Sourcing_Pattern.md) | +| **📦 Code Mgmt** | Branch Protection Rules | [ℹ️](./code_management/Branch_Protection_Rules.md) | [👆 View](./code_management/Branch_Protection_Rules.md) | +| | Code Review Guidelines | [ℹ️](./code_management/Code_Review_Guidelines.md) | [👆 View](./code_management/Code_Review_Guidelines.md) | +| | Dead Code Elimination | [ℹ️](./code_management/Dead_Code_Elimination.md) | [👆 View](./code_management/Dead_Code_Elimination.md) | +| | ESLint & Prettier Setup | [ℹ️](./code_management/ESLint_Prettier_Setup.md) | [👆 View](./code_management/ESLint_Prettier_Setup.md) | +| | Git Commit Convention | [ℹ️](./code_management/Git_Commit_Convention_Conventional_Commits.md) | [👆 View](./code_management/Git_Commit_Convention_Conventional_Commits.md) | +| | Git Hooks (Husky) | [ℹ️](./code_management/Git_Hooks_Husky.md) | [👆 View](./code_management/Git_Hooks_Husky.md) | +| | Managing Technical Debt | [ℹ️](./code_management/Managing_Technical_Debt.md) | [👆 View](./code_management/Managing_Technical_Debt.md) | +| | Monorepo Setup (Turborepo) | [ℹ️](./code_management/Monorepo_Setup_Turborepo.md) | [👆 View](./code_management/Monorepo_Setup_Turborepo.md) | +| | NPM Scripts Automation | [ℹ️](./code_management/Npm_Scripts_Automation.md) | [👆 View](./code_management/Npm_Scripts_Automation.md) | +| | Semantic Versioning | [ℹ️](./code_management/Semantic_Versioning_Strategy.md) | [👆 View](./code_management/Semantic_Versioning_Strategy.md) | +| | Git Workflow Strategies | [ℹ️](./code_management/Git_Workflow_Strategies.md) | [👆 View](./code_management/Git_Workflow_Strategies.md) | +| | Advanced Dependency Management (Dependabot/Renovate) | [ℹ️](./code_management/Advanced_Dependency_Management_Dependabot_Renovate.md) | [👆 View](./code_management/Advanced_Dependency_Management_Dependabot_Renovate.md) | +| | Code Documentation Standards (JSDoc/TSDoc) | [ℹ️](./code_management/Code_Documentation_Standards_JSDoc_TSDoc.md) | [👆 View](./code_management/Code_Documentation_Standards_JSDoc_TSDoc.md) | +| | Pull Request Templates & Standard Operating Procedure | [ℹ️](./code_management/Pull_Request_Templates_SOP.md) | [👆 View](./code_management/Pull_Request_Templates_SOP.md) | +| | Repository Structure & Organization | [ℹ️](./code_management/Repository_Structure_Organization.md) | [👆 View](./code_management/Repository_Structure_Organization.md) | +| | Dependency Pinning & Reproducible Builds | [ℹ️](./code_management/Dependency_Pinning_Reproducible_Builds.md) | [👆 View](./code_management/Dependency_Pinning_Reproducible_Builds.md) | +| | Git Rebase, Merge & Squash Guidelines | [ℹ️](./code_management/Git_Rebase_Merge_Squash_Guidelines.md) | [👆 View](./code_management/Git_Rebase_Merge_Squash_Guidelines.md) | +| | Pre-Commit & Pre-Push Hooks Automation | [ℹ️](./code_management/Pre-Commit_Pre-Push_Hooks_Automation.md) | [👆 View](./code_management/Pre-Commit_Pre-Push_Hooks_Automation.md) | +| | CHANGELOG Conventional Changelog Auto-Generation | [ℹ️](./code_management/CHANGELOG_Conventional_Changelog_Auto_Generation.md) | [👆 View](./code_management/CHANGELOG_Conventional_Changelog_Auto_Generation.md) | +| | Repository Archiving & Legacy Code Retirement | [ℹ️](./code_management/Repository_Archiving_Legacy_Code_Retirement.md) | [👆 View](./code_management/Repository_Archiving_Legacy_Code_Retirement.md) | +| **📝 Docs** | ADR Records | [ℹ️](./documentation/Architectural_Decision_Records_ADR.md) | [👆 View](./documentation/Architectural_Decision_Records_ADR.md) | +| | API Documentation | [ℹ️](./documentation/API_Documentation_Best_Practices.md) | [👆 View](./documentation/API_Documentation_Best_Practices.md) | +| | API Design Guidelines (REST) | [ℹ️](./documentation/API_Design_Guidelines_REST.md) | [👆 View](./documentation/API_Design_Guidelines_REST.md) | +| | Changelog Maintenance | [ℹ️](./documentation/Changelog_Maintenance.md) | [👆 View](./documentation/Changelog_Maintenance.md) | +| | Code Comments | [ℹ️](./documentation/Code_Comments_Best_Practices.md) | [👆 View](./documentation/Code_Comments_Best_Practices.md) | +| | Component Documentation (Storybook) | [ℹ️](./documentation/Component_Documentation_Storybook.md) | [👆 View](./documentation/Component_Documentation_Storybook.md) | +| | Contributing Guidelines | [ℹ️](./documentation/Contributing_Guidelines_CONTRIBUTING.md) | [👆 View](./documentation/Contributing_Guidelines_CONTRIBUTING.md) | +| | Diagramming with Mermaid.js | [ℹ️](./documentation/Diagramming_Mermaid_JS.md) | [👆 View](./documentation/Diagramming_Mermaid_JS.md) | +| | Effective User Documentation | [ℹ️](./documentation/Effective_User_Documentation.md) | [👆 View](./documentation/Effective_User_Documentation.md) | +| | Project Onboarding | [ℹ️](./documentation/Project_Onboarding_Guide.md) | [👆 View](./documentation/Project_Onboarding_Guide.md) | +| | Technical Spec Writing (RFC) | [ℹ️](./documentation/Technical_Spec_Writing.md) | [👆 View](./documentation/Technical_Spec_Writing.md) | +| | User Manual Creation | [ℹ️](./documentation/User_Manual_Creation.md) | [👆 View](./documentation/User_Manual_Creation.md) | +| | User Story Mapping | [ℹ️](./documentation/User_Story_Mapping.md) | [👆 View](./documentation/User_Story_Mapping.md) | +| | Writing Effective README | [ℹ️](./documentation/Writing_Effective_README.md) | [👆 View](./documentation/Writing_Effective_README.md) | diff --git a/TRAE-Skills/ai_engineering/AI_Agent_Design_Patterns.md b/TRAE-Skills/ai_engineering/AI_Agent_Design_Patterns.md new file mode 100644 index 0000000000000000000000000000000000000000..5cf8cb8ad92e3562136225bb2caa5e89704906f6 --- /dev/null +++ b/TRAE-Skills/ai_engineering/AI_Agent_Design_Patterns.md @@ -0,0 +1,99 @@ +# Skill: AI Agent Design Patterns + +## Purpose +To structure autonomous AI systems that can reason, plan, and execute tools to solve complex, multi-step problems using patterns like ReAct and Multi-Agent orchestration. + +## When to Use +- When the task requires multiple distinct steps (e.g., "Find the price of BTC and email me the summary"). +- When the LLM needs to interact with the outside world (APIs, Databases, Web Search). +- When the workflow is non-linear and depends on intermediate results. + +## Procedure + +### 1. Tool Definition (Zod-based) +Define the tools your agent can use with clear descriptions. + +```typescript +import { z } from "zod"; +import { tool } from "@langchain/core/tools"; + +const searchTool = tool( + async ({ query }) => { + // Implement search logic here + return `Results for ${query}...`; + }, + { + name: "web_search", + description: "Search the web for current events or technical info.", + schema: z.object({ + query: z.string(), + }), + } +); +``` + +### 2. The ReAct Agent Pattern +Implement the Reasoning + Acting loop. + +```typescript +import { ChatOpenAI } from "@langchain/openai"; +import { createReactAgent } from "@langchain/langgraph/prebuilt"; +import { MemorySaver } from "@langchain/langgraph"; + +const model = new ChatOpenAI({ modelName: "gpt-4o" }); +const tools = [searchTool]; +const checkpointer = new MemorySaver(); + +const app = createReactAgent({ + llm: model, + tools, + checkpointSaver: checkpointer, +}); + +// Usage +const result = await app.invoke( + { messages: [{ role: "user", content: "What is the current price of Ethereum?" }] }, + { configurable: { thread_id: "user_1" } } +); +``` + +### 3. Multi-Agent Orchestration (Hand-off) +Structure specialized agents that pass tasks to each other. + +```typescript +// Conceptual LangGraph Flow: +// 1. Router Agent -> Decides if it's a "Coding" or "Writing" task. +// 2. Coder Agent -> Generates code. +// 3. Reviewer Agent -> Reviews code. If errors, sends back to Coder. +// 4. Final Output. +``` + +### 4. Guardrails & Safety +Implement safety checks for tool execution. + +```typescript +const safeExecute = (action: string) => { + const forbidden = ["rm -rf", "delete", "drop table"]; + if (forbidden.some(word => action.includes(word))) { + throw new Error("Safety violation: forbidden command."); + } +}; +``` + +### 5. State Management +Maintain the conversation and tool execution state. + +```typescript +// Use LangGraph state to keep track of: +// - messages +// - tool_outputs +// - current_step +``` + +## Constraints +- **Infinite Loops**: Always set a `maxIterations` or recursion limit. +- **Context Bloat**: Agents generate a lot of tokens. Prune history or use summarization for long tasks. +- **Tool Descriptions**: The agent's performance is 90% dependent on how well you describe the tools. Be extremely precise. + +## Expected Output +A robust agentic system capable of autonomous problem solving by effectively utilizing provided tools. diff --git a/TRAE-Skills/ai_engineering/AI_Agents_LangGraph.md b/TRAE-Skills/ai_engineering/AI_Agents_LangGraph.md new file mode 100644 index 0000000000000000000000000000000000000000..8f43e07e6416e52fa79ccde0e4259ccdc0eb41c2 --- /dev/null +++ b/TRAE-Skills/ai_engineering/AI_Agents_LangGraph.md @@ -0,0 +1,232 @@ +# Skill: Building AI Agents with LangGraph + +## Purpose +To build stateful, multi-agent, and agentic systems using LangGraph for complex workflows (reasoning, tool use, multi-step tasks). + +## When to Use +- When building customer support agents that use multiple tools +- For research assistants that can browse, analyze, and summarize +- When creating multi-agent collaboration systems +- For building RAG with reasoning loops +- When you need agentic workflows with human-in-the-loop + +## Procedure + +### 1. Basic LangGraph Agent +Create a simple agent with tools. + +```python +from typing import Annotated, Literal, TypedDict +from langchain_openai import ChatOpenAI +from langchain_core.tools import tool +from langgraph.graph import StateGraph, START, END +from langgraph.graph.message import add_messages +from langgraph.prebuilt import ToolNode + +# Define tools +@tool +def search_web(query: str) -> str: + """Search the web for a query.""" + return f"Search results for '{query}': LangGraph is a library for building stateful agents." + +@tool +def calculate(expression: str) -> str: + """Calculate a mathematical expression.""" + try: + return str(eval(expression)) + except: + return "Invalid expression" + +tools = [search_web, calculate] + +# Define state +class AgentState(TypedDict): + messages: Annotated[list, add_messages] + +# Initialize model +llm = ChatOpenAI(model="gpt-4o", temperature=0) +llm_with_tools = llm.bind_tools(tools) + +# Define nodes +def agent_node(state: AgentState): + response = llm_with_tools.invoke(state["messages"]) + return {"messages": [response]} + +tool_node = ToolNode(tools) + +# Define conditional edge +def should_continue(state: AgentState) -> Literal["tools", END]: + last_message = state["messages"][-1] + if last_message.tool_calls: + return "tools" + return END + +# Build graph +graph = StateGraph(AgentState) +graph.add_node("agent", agent_node) +graph.add_node("tools", tool_node) +graph.add_edge(START, "agent") +graph.add_conditional_edges("agent", should_continue) +graph.add_edge("tools", "agent") + +app = graph.compile() + +# Run agent +result = app.invoke({"messages": [("user", "What's 25 * 4 + 10? Also, tell me about LangGraph.")]}) +print(result["messages"][-1].content) +``` + +### 2. Multi-Agent Collaboration +Build a team of specialized agents. + +```python +from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder +from langgraph.graph import StateGraph, START, END + +# Define specialized agents +researcher_prompt = ChatPromptTemplate.from_messages([ + ("system", "You are a researcher. Use the search tool to find information."), + MessagesPlaceholder(variable_name="messages") +]) + +writer_prompt = ChatPromptTemplate.from_messages([ + ("system", "You are a writer. Take research and write a concise summary."), + MessagesPlaceholder(variable_name="messages") +]) + +reviewer_prompt = ChatPromptTemplate.from_messages([ + ("system", "You are a reviewer. Check the summary and improve it if needed."), + MessagesPlaceholder(variable_name="messages") +]) + +class TeamState(TypedDict): + messages: Annotated[list, add_messages] + current_agent: str + final_summary: str + +def researcher_node(state: TeamState): + researcher_llm = researcher_prompt | llm_with_tools + response = researcher_llm.invoke(state["messages"]) + return {"messages": [response], "current_agent": "researcher"} + +def writer_node(state: TeamState): + writer_llm = writer_prompt | llm + response = writer_llm.invoke(state["messages"]) + return {"messages": [response], "current_agent": "writer", "final_summary": response.content} + +def reviewer_node(state: TeamState): + reviewer_llm = reviewer_prompt | llm + response = reviewer_llm.invoke(state["messages"]) + return {"messages": [response], "current_agent": "reviewer", "final_summary": response.content} + +def route_team(state: TeamState) -> Literal["researcher", "writer", "reviewer", END]: + if not state["current_agent"]: + return "researcher" + elif state["current_agent"] == "researcher": + return "writer" + elif state["current_agent"] == "writer": + return "reviewer" + return END + +team_graph = StateGraph(TeamState) +team_graph.add_node("researcher", researcher_node) +team_graph.add_node("writer", writer_node) +team_graph.add_node("reviewer", reviewer_node) +team_graph.add_conditional_edges(START, route_team) +team_graph.add_conditional_edges("researcher", route_team) +team_graph.add_conditional_edges("writer", route_team) + +team_app = team_graph.compile() + +# Run team +result = team_app.invoke({"messages": [("user", "Research LangGraph and write a summary.")]}) +print("Final Summary:", result["final_summary"]) +``` + +### 3. Human-in-the-Loop (HITL) +Add human approval steps. + +```python +from langgraph.checkpoint.memory import MemorySaver + +class HITLState(TypedDict): + messages: Annotated[list, add_messages] + approved: bool + +def agent_node_with_approval(state: HITLState): + if not state.get("approved", False): + # Wait for human approval (interrupt) + pass + response = llm_with_tools.invoke(state["messages"]) + return {"messages": [response]} + +checkpointer = MemorySaver() + +hitl_graph = StateGraph(HITLState) +hitl_graph.add_node("agent", agent_node_with_approval) +hitl_graph.add_edge(START, "agent") +hitl_graph.add_conditional_edges("agent", should_continue) +hitl_graph.add_edge("tools", "agent") + +hitl_app = hitl_graph.compile(checkpointer=checkpointer, interrupt_before=["agent"]) + +config = {"configurable": {"thread_id": "1"}} + +# Initial run (interrupts before agent) +initial_result = hitl_app.invoke({"messages": [("user", "Approve this action?")]}, config) + +# Human approves +human_approved_state = hitl_app.update_state(config, {"approved": True}) + +# Continue execution +final_result = hitl_app.invoke(None, config) +``` + +### 4. RAG Agent with LangGraph +Build an agent that does RAG with reasoning. + +```python +from langchain_community.vectorstores import InMemoryVectorStore +from langchain_openai import OpenAIEmbeddings +from langchain_core.documents import Document + +# Sample documents +documents = [ + Document(page_content="LangGraph is for building stateful agents.", metadata={"source": "doc1"}), + Document(page_content="Agents can use tools and have memory.", metadata={"source": "doc2"}) +] + +vector_store = InMemoryVectorStore.from_documents(documents, OpenAIEmbeddings()) +retriever = vector_store.as_retriever(k=2) + +@tool +def retrieve_documents(query: str) -> str: + """Retrieve relevant documents from the knowledge base.""" + docs = retriever.invoke(query) + return "\n\n".join([f"Source: {d.metadata['source']}\n{d.page_content}" for d in docs]) + +rag_tools = [retrieve_documents] +rag_llm = llm.bind_tools(rag_tools) + +# RAG agent graph +rag_graph = StateGraph(AgentState) +rag_graph.add_node("agent", lambda s: {"messages": [rag_llm.invoke(s["messages"])]}) +rag_graph.add_node("tools", ToolNode(rag_tools)) +rag_graph.add_edge(START, "agent") +rag_graph.add_conditional_edges("agent", should_continue) +rag_graph.add_edge("tools", "agent") + +rag_app = rag_graph.compile() + +result = rag_app.invoke({"messages": [("user", "What is LangGraph used for?")]}) +print(result["messages"][-1].content) +``` + +## Best Practices +- **State Design**: Keep state minimal and typed +- **Tool Design**: Make tools with clear, specific descriptions +- **Error Handling**: Add fallback edges for failures +- **Checkpoints**: Use checkpoints for long-running workflows +- **Evaluation**: Test agent workflows with LangSmith +- **Cost**: Limit tool calls to control costs +- **Human-in-the-Loop**: Add approval steps for high-stakes actions diff --git a/TRAE-Skills/ai_engineering/AI_Experiment_Tracking.md b/TRAE-Skills/ai_engineering/AI_Experiment_Tracking.md new file mode 100644 index 0000000000000000000000000000000000000000..2177581a8b81ec6eabeb843ab31f34b1e7efffcf --- /dev/null +++ b/TRAE-Skills/ai_engineering/AI_Experiment_Tracking.md @@ -0,0 +1,640 @@ +# Skill: AI Experiment Tracking + +## Purpose +To systematically track machine learning experiments, including hyperparameters, metrics, artifacts, and results for reproducibility and optimization. + +## When to Use +- When running multiple ML experiments with different configurations +- When comparing model performance across different approaches +- When needing to reproduce experimental results +- When optimizing hyperparameters and model architectures + +## Procedure + +### 1. Experiment Tracking Framework +Create a comprehensive experiment tracking system. + +```python +import json +import logging +from datetime import datetime +from typing import Any, Dict, List, Optional +from pathlib import Path +import hashlib +import pandas as pd +import matplotlib.pyplot as plt + +class ExperimentTracker: + """Track machine learning experiments.""" + + def __init__(self, project_name: str, base_dir: str = "./experiments"): + self.project_name = project_name + self.base_dir = Path(base_dir) + self.project_dir = self.base_dir / project_name + self.project_dir.mkdir(parents=True, exist_ok=True) + + self.current_experiment = None + self.logger = logging.getLogger(f"ExperimentTracker.{project_name}") + + # Initialize experiment registry + self.registry_file = self.project_dir / "experiments_registry.json" + self._init_registry() + + def _init_registry(self): + """Initialize experiments registry.""" + if not self.registry_file.exists(): + self._save_registry({"experiments": [], "total": 0}) + + def _load_registry(self) -> Dict[str, Any]: + """Load experiments registry.""" + with open(self.registry_file, 'r') as f: + return json.load(f) + + def _save_registry(self, registry: Dict[str, Any]): + """Save experiments registry.""" + with open(self.registry_file, 'w') as f: + json.dump(registry, f, indent=2) + + def start_experiment(self, name: str, description: str = "", tags: List[str] = None) -> str: + """Start a new experiment.""" + experiment_id = self._generate_experiment_id(name) + + experiment = { + "id": experiment_id, + "name": name, + "description": description, + "tags": tags or [], + "start_time": datetime.now().isoformat(), + "end_time": None, + "status": "running", + "hyperparameters": {}, + "metrics": {}, + "artifacts": [], + "metadata": {} + } + + # Create experiment directory + experiment_dir = self.project_dir / experiment_id + experiment_dir.mkdir(exist_ok=True) + + # Update registry + registry = self._load_registry() + registry["experiments"].append(experiment) + registry["total"] += 1 + self._save_registry(registry) + + self.current_experiment = experiment + self.logger.info(f"Started experiment: {name} (ID: {experiment_id})") + + return experiment_id + + def log_hyperparameters(self, params: Dict[str, Any]): + """Log hyperparameters for current experiment.""" + if not self.current_experiment: + raise ValueError("No active experiment. Call start_experiment() first.") + + self.current_experiment["hyperparameters"].update(params) + self._update_current_experiment() + self.logger.info(f"Logged hyperparameters: {params}") + + def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None): + """Log metrics for current experiment.""" + if not self.current_experiment: + raise ValueError("No active experiment. Call start_experiment() first.") + + if step is not None: + if "metric_history" not in self.current_experiment: + self.current_experiment["metric_history"] = {} + + for metric_name, value in metrics.items(): + if metric_name not in self.current_experiment["metric_history"]: + self.current_experiment["metric_history"][metric_name] = [] + + self.current_experiment["metric_history"][metric_name].append({ + "step": step, + "value": value, + "timestamp": datetime.now().isoformat() + }) + + self.current_experiment["metrics"].update(metrics) + self._update_current_experiment() + self.logger.info(f"Logged metrics: {metrics}") + + def log_artifact(self, artifact_path: str, artifact_type: str = "file"): + """Log an artifact for current experiment.""" + if not self.current_experiment: + raise ValueError("No active experiment. Call start_experiment() first.") + + import shutil + experiment_dir = self.project_dir / self.current_experiment["id"] + artifact_name = Path(artifact_path).name + dest_path = experiment_dir / "artifacts" / artifact_name + + # Create artifacts directory + dest_path.parent.mkdir(exist_ok=True) + + # Copy artifact + shutil.copy2(artifact_path, dest_path) + + artifact_info = { + "name": artifact_name, + "type": artifact_type, + "original_path": artifact_path, + "stored_path": str(dest_path), + "timestamp": datetime.now().isoformat() + } + + self.current_experiment["artifacts"].append(artifact_info) + self._update_current_experiment() + self.logger.info(f"Logged artifact: {artifact_name}") + + def end_experiment(self, status: str = "completed"): + """End the current experiment.""" + if not self.current_experiment: + raise ValueError("No active experiment to end.") + + self.current_experiment["end_time"] = datetime.now().isoformat() + self.current_experiment["status"] = status + self._update_current_experiment() + + self.logger.info(f"Ended experiment: {self.current_experiment['name']} (status: {status})") + self.current_experiment = None + + def _update_current_experiment(self): + """Update current experiment in registry.""" + if not self.current_experiment: + return + + registry = self._load_registry() + for i, exp in enumerate(registry["experiments"]): + if exp["id"] == self.current_experiment["id"]: + registry["experiments"][i] = self.current_experiment + break + + self._save_registry(registry) + + def _generate_experiment_id(self, name: str) -> str: + """Generate unique experiment ID.""" + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + name_hash = hashlib.md5(name.encode()).hexdigest()[:8] + return f"exp_{timestamp}_{name_hash}" + + def get_experiment(self, experiment_id: str) -> Optional[Dict[str, Any]]: + """Get experiment by ID.""" + registry = self._load_registry() + for exp in registry["experiments"]: + if exp["id"] == experiment_id: + return exp + return None + + def list_experiments(self, status: Optional[str] = None, tags: Optional[List[str]] = None) -> List[Dict[str, Any]]: + """List experiments with optional filtering.""" + registry = self._load_registry() + experiments = registry["experiments"] + + if status: + experiments = [exp for exp in experiments if exp["status"] == status] + + if tags: + experiments = [exp for exp in experiments if any(tag in exp["tags"] for tag in tags)] + + return experiments + + def compare_experiments(self, experiment_ids: List[str]) -> pd.DataFrame: + """Compare experiments side by side.""" + experiments = [] + for exp_id in experiment_ids: + exp = self.get_experiment(exp_id) + if exp: + experiments.append(exp) + + if not experiments: + return pd.DataFrame() + + comparison_data = [] + for exp in experiments: + row = { + "id": exp["id"], + "name": exp["name"], + "status": exp["status"], + **exp["hyperparameters"], + **exp["metrics"] + } + comparison_data.append(row) + + return pd.DataFrame(comparison_data) +``` + +### 2. Automated Hyperparameter Logging +Automatically track hyperparameters. + +```python +from functools import wraps + +def track_hyperparameters(tracker: ExperimentTracker): + """Decorator to automatically track function parameters as hyperparameters.""" + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + # Extract hyperparameters from function arguments + import inspect + sig = inspect.signature(func) + bound_args = sig.bind(*args, **kwargs) + bound_args.apply_defaults() + + # Log all parameters as hyperparameters + params = dict(bound_args.arguments) + # Remove self parameter if present + params.pop('self', None) + params.pop('tracker', None) + + tracker.log_hyperparameters(params) + + return func(*args, **kwargs) + return wrapper + return decorator + +# Usage +# @track_hyperparameters(tracker) +# def train_model(learning_rate, batch_size, epochs, model_type): +# # Training code here +# pass +``` + +### 3. Metrics Visualization +Create visualizations for experiment comparison. + +```python +class ExperimentVisualizer: + """Visualize experiment results.""" + + def __init__(self, tracker: ExperimentTracker): + self.tracker = tracker + + def plot_metric_comparison(self, metric_name: str, top_n: int = 10): + """Plot metric comparison across experiments.""" + experiments = self.tracker.list_experiments(status="completed") + + # Filter experiments that have the metric + valid_experiments = [ + exp for exp in experiments + if metric_name in exp["metrics"] + ] + + if not valid_experiments: + print(f"No experiments found with metric: {metric_name}") + return + + # Sort by metric value + valid_experiments.sort(key=lambda x: x["metrics"][metric_name]) + + # Take top N + top_experiments = valid_experiments[:top_n] + + # Create plot + plt.figure(figsize=(12, 6)) + names = [exp["name"] for exp in top_experiments] + values = [exp["metrics"][metric_name] for exp in top_experiments] + + plt.bar(range(len(names)), values) + plt.xticks(range(len(names)), names, rotation=45, ha='right') + plt.ylabel(metric_name) + plt.title(f'{metric_name} Comparison (Top {top_n})') + plt.tight_layout() + + # Save plot + output_path = self.tracker.project_dir / f"{metric_name}_comparison.png" + plt.savefig(output_path) + plt.close() + + print(f"Plot saved to: {output_path}") + + def plot_training_curves(self, experiment_id: str, metric_name: str): + """Plot training curves for a specific experiment.""" + exp = self.tracker.get_experiment(experiment_id) + + if not exp or "metric_history" not in exp: + print(f"No metric history found for experiment: {experiment_id}") + return + + if metric_name not in exp["metric_history"]: + print(f"Metric {metric_name} not found in experiment history") + return + + # Extract data + history = exp["metric_history"][metric_name] + steps = [entry["step"] for entry in history] + values = [entry["value"] for entry in history] + + # Create plot + plt.figure(figsize=(10, 6)) + plt.plot(steps, values, marker='o') + plt.xlabel("Step") + plt.ylabel(metric_name) + plt.title(f'{metric_name} Training Curve - {exp["name"]}') + plt.grid(True) + + # Save plot + output_path = self.tracker.project_dir / f"{experiment_id}_{metric_name}_curve.png" + plt.savefig(output_path) + plt.close() + + print(f"Training curve saved to: {output_path}") + + def create_experiment_report(self, experiment_id: str) -> str: + """Create a comprehensive experiment report.""" + exp = self.tracker.get_experiment(experiment_id) + + if not exp: + return f"Experiment not found: {experiment_id}" + + report = f""" +Experiment Report +{'=' * 50} + +Name: {exp['name']} +ID: {exp['id']} +Status: {exp['status']} +Description: {exp['description']} + +Timeline: +- Started: {exp['start_time']} +- Ended: {exp['end_time'] or 'Running'} + +Tags: {', '.join(exp['tags']) if exp['tags'] else 'None'} + +Hyperparameters: +""" + for param, value in exp['hyperparameters'].items(): + report += f"- {param}: {value}\n" + + report += f"\nFinal Metrics:\n" + for metric, value in exp['metrics'].items(): + report += f"- {metric}: {value}\n" + + if exp['artifacts']: + report += f"\nArtifacts ({len(exp['artifacts'])}):\n" + for artifact in exp['artifacts']: + report += f"- {artifact['name']} ({artifact['type']})\n" + + return report +``` + +### 4. Model Artifact Management +Manage model artifacts and versions. + +```python +import joblib +import pickle + +class ModelArtifactManager: + """Manage model artifacts with versioning.""" + + def __init__(self, tracker: ExperimentTracker): + self.tracker = tracker + self.artifacts_dir = tracker.project_dir / "artifacts" + self.artifacts_dir.mkdir(exist_ok=True) + + def save_model(self, model, model_name: str, framework: str = "sklearn"): + """Save model and log as artifact.""" + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + artifact_filename = f"{model_name}_{timestamp}.joblib" + artifact_path = self.artifacts_dir / artifact_filename + + # Save model + if framework == "sklearn": + joblib.dump(model, artifact_path) + elif framework == "pickle": + with open(artifact_path, 'wb') as f: + pickle.dump(model, f) + else: + raise ValueError(f"Unsupported framework: {framework}") + + # Log artifact + self.tracker.log_artifact(str(artifact_path), artifact_type="model") + + return artifact_path + + def load_model(self, artifact_path: str): + """Load model from artifact.""" + if artifact_path.endswith('.joblib'): + return joblib.load(artifact_path) + elif artifact_path.endswith('.pkl'): + with open(artifact_path, 'rb') as f: + return pickle.load(f) + else: + raise ValueError(f"Unsupported artifact format: {artifact_path}") + + def compare_model_versions(self, model_name: str) -> pd.DataFrame: + """Compare different versions of a model.""" + experiments = self.tracker.list_experiments() + + model_versions = [] + for exp in experiments: + for artifact in exp['artifacts']: + if artifact['name'].startswith(model_name) and artifact['type'] == 'model': + model_versions.append({ + 'experiment_id': exp['id'], + 'experiment_name': exp['name'], + 'artifact_name': artifact['name'], + 'created': artifact['timestamp'], + **exp['metrics'] + }) + + return pd.DataFrame(model_versions) +``` + +### 5. Experiment Analysis and Insights +Analyze experiments to derive insights. + +```python +class ExperimentAnalyzer: + """Analyze experiments to provide insights.""" + + def __init__(self, tracker: ExperimentTracker): + self.tracker = tracker + + def find_best_experiment(self, metric_name: str, higher_is_better: bool = True) -> Optional[Dict[str, Any]]: + """Find the best performing experiment for a given metric.""" + experiments = self.tracker.list_experiments(status="completed") + + valid_experiments = [ + exp for exp in experiments + if metric_name in exp["metrics"] + ] + + if not valid_experiments: + return None + + if higher_is_better: + best_exp = max(valid_experiments, key=lambda x: x["metrics"][metric_name]) + else: + best_exp = min(valid_experiments, key=lambda x: x["metrics"][metric_name]) + + return best_exp + + def analyze_hyperparameter_importance(self, metric_name: str) -> pd.DataFrame: + """Analyze the importance of hyperparameters on a metric.""" + experiments = self.tracker.list_experiments(status="completed") + + valid_experiments = [ + exp for exp in experiments + if metric_name in exp["metrics"] + ] + + if not valid_experiments: + return pd.DataFrame() + + # Create analysis data + analysis_data = [] + for exp in valid_experiments: + row = { + "experiment_id": exp["id"], + "experiment_name": exp["name"], + "metric_value": exp["metrics"][metric_name] + } + # Add all hyperparameters + row.update(exp["hyperparameters"]) + analysis_data.append(row) + + df = pd.DataFrame(analysis_data) + + # Calculate correlations for numeric hyperparameters + numeric_cols = df.select_dtypes(include=['number']).columns + if len(numeric_cols) > 1: + correlations = df[numeric_cols].corr()['metric_value'].sort_values(ascending=False) + return correlations + + return df + + def generate_experiment_summary(self) -> str: + """Generate a summary of all experiments.""" + experiments = self.tracker.list_experiments() + + summary = f""" +Experiment Summary for {self.tracker.project_name} +{'=' * 60} + +Total Experiments: {len(experiments)} + +Status Breakdown: +""" + status_counts = {} + for exp in experiments: + status = exp["status"] + status_counts[status] = status_counts.get(status, 0) + 1 + + for status, count in sorted(status_counts.items()): + summary += f"- {status}: {count}\n" + + if status_counts.get("completed", 0) > 0: + summary += f"\nCompleted Experiments: {status_counts['completed']}\n" + summary += "Best performing experiments by common metrics:\n" + + common_metrics = {} + for exp in experiments: + if exp["status"] == "completed": + for metric in exp["metrics"].keys(): + if metric not in common_metrics: + common_metrics[metric] = [] + common_metrics[metric].append((exp["name"], exp["metrics"][metric])) + + for metric, values in common_metrics.items(): + best = max(values, key=lambda x: x[1]) + summary += f"- {metric}: {best[0]} ({best[1]:.4f})\n" + + return summary +``` + +### 6. Complete Example Usage +Demonstrate complete experiment tracking workflow. + +```python +def example_experiment_tracking(): + """Demonstrate complete experiment tracking.""" + + # Initialize tracker + tracker = ExperimentTracker(project_name="sentiment_analysis") + + # Start experiment + exp_id = tracker.start_experiment( + name="random_forest_baseline", + description="Random Forest baseline model for sentiment analysis", + tags=["baseline", "random_forest"] + ) + + # Log hyperparameters + tracker.log_hyperparameters({ + "model_type": "random_forest", + "n_estimators": 100, + "max_depth": 10, + "min_samples_split": 2, + "learning_rate": None + }) + + # Simulate training and log metrics + import numpy as np + for epoch in range(10): + # Simulate training + train_loss = 1.0 - (epoch * 0.08) + val_loss = 1.0 - (epoch * 0.07) + np.random.normal(0, 0.05) + + tracker.log_metrics({ + "train_loss": train_loss, + "val_loss": val_loss, + "epoch": epoch + }, step=epoch) + + # Log final metrics + tracker.log_metrics({ + "train_accuracy": 0.92, + "val_accuracy": 0.87, + "test_accuracy": 0.85, + "f1_score": 0.84, + "precision": 0.86, + "recall": 0.82 + }) + + # Save model artifact + artifact_manager = ModelArtifactManager(tracker) + import joblib + dummy_model = {"model": "dummy_model", "accuracy": 0.85} + joblib.dump(dummy_model, "dummy_model.joblib") + artifact_manager.save_model(dummy_model, "sentiment_model") + + # End experiment + tracker.end_experiment(status="completed") + + # Create visualizations + visualizer = ExperimentVisualizer(tracker) + visualizer.plot_metric_comparison("test_accuracy") + visualizer.plot_training_curves(exp_id, "train_loss") + + # Generate report + report = visualizer.create_experiment_report(exp_id) + print(report) + + # Analyze experiments + analyzer = ExperimentAnalyzer(tracker) + best_exp = analyzer.find_best_experiment("test_accuracy") + print(f"\nBest experiment: {best_exp['name']} with accuracy {best_exp['metrics']['test_accuracy']}") + + summary = analyzer.generate_experiment_summary() + print(summary) + +# Usage +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + example_experiment_tracking() +``` + +## Constraints +- **Storage Space**: Experiment artifacts can consume significant storage +- **Performance**: Extensive logging may impact training performance +- **Reproducibility**: Ensure complete environment capture for true reproducibility +- **Scalability**: Consider scalability for large numbers of experiments +- **Privacy**: Be careful with sensitive data in experiment logs +- **Organization**: Maintain consistent naming and tagging conventions + +## Expected Output +Comprehensive experiment tracking system that captures all aspects of ML experiments, enables detailed comparison and analysis, and provides insights for model optimization. diff --git a/TRAE-Skills/ai_engineering/AI_Model_Evaluation.md b/TRAE-Skills/ai_engineering/AI_Model_Evaluation.md new file mode 100644 index 0000000000000000000000000000000000000000..271971b3e84a63f833868f2685e2c0f8ba1689eb --- /dev/null +++ b/TRAE-Skills/ai_engineering/AI_Model_Evaluation.md @@ -0,0 +1,107 @@ +# Skill: AI Model Evaluation + +## Purpose +To systematically assess the performance, accuracy, and safety of LLM outputs using quantitative metrics and "LLM-as-a-Judge" patterns, ensuring production readiness. + +## When to Use +- Before deploying any LLM application to production. +- When comparing different models (e.g., GPT-4o vs. Claude 3.5 Sonnet) or prompt versions. +- To detect regressions after updating prompts or RAG knowledge bases. + +## Procedure + +### 1. Define the Evaluation Dataset (Golden Set) +Create a `tests.json` file containing inputs and expected outputs. + +```json +[ + { + "input": "What is the return policy?", + "expected": "You can return items within 30 days.", + "context": "Our policy allows returns for 30 days from purchase date." + } +] +``` + +### 2. Implementation with Promptfoo (CLI) +Promptfoo is a popular tool for running batch evaluations. + +```bash +# Install +npm install -g promptfoo + +# Initialize +promptfoo init +``` + +Configure `promptfooconfig.yaml`: +```yaml +prompts: + - "Answer this question using the context: {{context}}. Question: {{input}}" + +providers: + - openai:gpt-4o + +tests: + - vars: + input: "What is the return policy?" + context: "30-day return policy applies." + assert: + - type: icontains + value: "30 days" + - type: llm-rubric + value: "Does not mention unrelated topics" +``` + +### 3. RAG Evaluation (Ragas/DeepEval) +For RAG systems, evaluate the three-way relationship: Question, Context, and Answer. + +```typescript +import { rce } from "deepeval"; // Conceptual example + +async function evaluateRag(query: string, retrievalContext: string, output: string) { + // 1. Faithfulness: Is the answer grounded in the context? + // 2. Answer Relevance: Does it answer the query? + // 3. Context Precision: Was the retrieved context relevant? +} +``` + +### 4. Custom LLM-as-a-Judge Script +Use a stronger model to grade your target model. + +```typescript +async function gradeOutput(question: string, answer: string, reference: string) { + const graderPrompt = ` + You are an impartial judge. Grade the student's answer based on the reference. + Question: ${question} + Reference: ${reference} + Student Answer: ${answer} + + Provide a score from 1-10 and a brief explanation. + Output JSON: { "score": number, "explanation": string } + `; + + // Call GPT-4 with JSON mode enabled +} +``` + +### 5. Continuous Integration (CI) +Integrate evaluation into your GitHub Actions to prevent regressions. + +```yaml +# .github/workflows/ai-eval.yml +jobs: + evaluate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: npx promptfoo eval +``` + +## Constraints +- **Bias**: LLM judges tend to prefer longer answers or answers from the same provider. Use diverse judges (OpenAI + Anthropic). +- **Cost**: Running evaluations on 1000s of rows can be expensive. Use `gpt-4o-mini` for simpler checks. +- **Reference Accuracy**: A "Golden Set" is only as good as the human-verified reference answers. + +## Expected Output +A detailed report (HTML/JSON) showing pass/fail status, accuracy percentages, and regression analysis. diff --git a/TRAE-Skills/ai_engineering/AI_Model_Serving.md b/TRAE-Skills/ai_engineering/AI_Model_Serving.md new file mode 100644 index 0000000000000000000000000000000000000000..289f6dbb7d88f1e8651f8019250254edb35d52a7 --- /dev/null +++ b/TRAE-Skills/ai_engineering/AI_Model_Serving.md @@ -0,0 +1,503 @@ +# Skill: AI Model Serving + +## Purpose +To deploy and serve machine learning models in production environments with proper scaling, monitoring, and API integration. + +## When to Use +- When deploying ML models to production +- When building ML-powered APIs and services +- When implementing real-time inference systems +- When scaling ML services for production traffic + +## Procedure + +### 1. Model Server Setup +Create a production-ready model server. + +```python +from fastapi import FastAPI, HTTPException, BackgroundTasks +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel, Field +from typing import List, Optional, Dict, Any +import joblib +import logging +import time +from datetime import datetime +import asyncio +from prometheus_client import Counter, Histogram, generate_latest +import numpy as np + +# Define request/response models +class PredictionRequest(BaseModel): + model_name: str = Field(..., description="Name of the model to use") + model_version: str = Field(default="latest", description="Version of the model") + input_data: Dict[str, Any] = Field(..., description="Input data for prediction") + preprocessing: Optional[Dict[str, Any]] = Field(default=None, description="Preprocessing options") + postprocessing: Optional[Dict[str, Any]] = Field(default=None, description="Postprocessing options") + +class PredictionResponse(BaseModel): + prediction: Any + model_name: str + model_version: str + prediction_time: float + timestamp: str + metadata: Optional[Dict[str, Any]] = None + +class HealthResponse(BaseModel): + status: str + models_loaded: List[str] + uptime_seconds: float + version: str + +# Create FastAPI app +app = FastAPI( + title="ML Model Serving API", + description="Production ML model inference server", + version="1.0.0" +) + +# Add CORS middleware +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Prometheus metrics +prediction_counter = Counter('predictions_total', 'Total predictions', ['model_name', 'status']) +prediction_duration = Histogram('prediction_duration_seconds', 'Prediction duration', ['model_name']) + +class ModelServer: + """Production model server.""" + + def __init__(self, model_registry_path: str = "./models"): + self.model_registry_path = model_registry_path + self.loaded_models = {} + self.model_metadata = {} + self.start_time = time.time() + self.logger = logging.getLogger("ModelServer") + + # Load model registry + self._load_model_registry() + + def _load_model_registry(self): + """Load model registry.""" + import json + registry_path = f"{self.model_registry_path}/model_registry.json" + try: + with open(registry_path, 'r') as f: + self.model_registry = json.load(f) + except FileNotFoundError: + self.model_registry = {"models": [], "current": None} + self.logger.warning("Model registry not found, starting with empty registry") + + def load_model(self, model_name: str, version: str = "latest"): + """Load model into memory.""" + model_key = f"{model_name}_{version}" + + if model_key in self.loaded_models: + return self.loaded_models[model_key] + + # Find model in registry + model_info = None + for model in self.model_registry["models"]: + if model["model_type"] == model_name: + if version == "latest" or model["version"] == version: + model_info = model + break + + if not model_info: + raise ValueError(f"Model {model_name} version {version} not found") + + # Load model from disk + model_path = model_info["model_path"] + try: + model = joblib.load(model_path) + self.loaded_models[model_key] = model + self.model_metadata[model_key] = model_info + self.logger.info(f"Loaded model: {model_key}") + return model + except Exception as e: + self.logger.error(f"Failed to load model {model_key}: {str(e)}") + raise + + def unload_model(self, model_name: str, version: str = "latest"): + """Unload model from memory.""" + model_key = f"{model_name}_{version}" + if model_key in self.loaded_models: + del self.loaded_models[model_key] + del self.model_metadata[model_key] + self.logger.info(f"Unloaded model: {model_key}") + + async def predict(self, request: PredictionRequest) -> PredictionResponse: + """Make prediction.""" + start_time = time.time() + model_key = f"{request.model_name}_{request.model_version}" + + try: + # Load model if not in memory + model = self.load_model(request.model_name, request.model_version) + + # Preprocess input + processed_input = self._preprocess_input(request.input_data, request.preprocessing) + + # Make prediction + prediction = model.predict([processed_input])[0] if hasattr(model, 'predict') else model(processed_input) + + # Postprocess prediction + final_prediction = self._postprocess_prediction(prediction, request.postprocessing) + + prediction_time = time.time() - start_time + + # Record metrics + prediction_counter.labels(model_name=request.model_name, status='success').inc() + prediction_duration.labels(model_name=request.model_name).observe(prediction_time) + + return PredictionResponse( + prediction=final_prediction, + model_name=request.model_name, + model_version=request.model_version, + prediction_time=prediction_time, + timestamp=datetime.now().isoformat(), + metadata=self.model_metadata.get(model_key) + ) + + except Exception as e: + prediction_counter.labels(model_name=request.model_name, status='error').inc() + self.logger.error(f"Prediction failed: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + def _preprocess_input(self, input_data: Dict[str, Any], preprocessing: Optional[Dict[str, Any]]) -> Any: + """Preprocess input data.""" + # Implement preprocessing logic based on model requirements + # This is a placeholder - customize based on your model + return input_data + + def _postprocess_prediction(self, prediction: Any, postprocessing: Optional[Dict[str, Any]]) -> Any: + """Postprocess prediction.""" + # Implement postprocessing logic + return prediction + + def get_health(self) -> HealthResponse: + """Get server health status.""" + return HealthResponse( + status="healthy", + models_loaded=list(self.loaded_models.keys()), + uptime_seconds=time.time() - self.start_time, + version="1.0.0" + ) + +# Create model server instance +model_server = ModelServer() + +# API endpoints +@app.post("/predict", response_model=PredictionResponse) +async def predict(request: PredictionRequest): + """Make prediction.""" + return await model_server.predict(request) + +@app.get("/health", response_model=HealthResponse) +async def health(): + """Health check endpoint.""" + return model_server.get_health() + +@app.get("/models") +async def list_models(): + """List available models.""" + return {"models": model_server.model_registry["models"]} + +@app.post("/models/{model_name}/load") +async def load_model_endpoint(model_name: str, version: str = "latest"): + """Load model endpoint.""" + try: + model_server.load_model(model_name, version) + return {"status": "success", "message": f"Model {model_name} version {version} loaded"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + +@app.delete("/models/{model_name}/unload") +async def unload_model_endpoint(model_name: str, version: str = "latest"): + """Unload model endpoint.""" + model_server.unload_model(model_name, version) + return {"status": "success", "message": f"Model {model_name} version {version} unloaded"} + +@app.get("/metrics") +async def metrics(): + """Prometheus metrics endpoint.""" + return generate_latest() +``` + +### 2. Batch Prediction Service +Handle batch prediction requests efficiently. + +```python +from concurrent.futures import ThreadPoolExecutor +import asyncio +from typing import List + +class BatchPredictionService: + """Service for batch predictions.""" + + def __init__(self, model_server: ModelServer, max_workers: int = 4): + self.model_server = model_server + self.executor = ThreadPoolExecutor(max_workers=max_workers) + self.logger = logging.getLogger("BatchPredictionService") + + async def predict_batch(self, requests: List[PredictionRequest]) -> List[PredictionResponse]: + """Process multiple prediction requests in parallel.""" + loop = asyncio.get_event_loop() + + # Create tasks for parallel processing + tasks = [ + loop.run_in_executor( + self.executor, + self._predict_sync, + request + ) + for request in requests + ] + + # Wait for all tasks to complete + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Handle exceptions + responses = [] + for i, result in enumerate(results): + if isinstance(result, Exception): + self.logger.error(f"Batch prediction failed for request {i}: {str(result)}") + # Create error response + responses.append(PredictionResponse( + prediction=None, + model_name=requests[i].model_name, + model_version=requests[i].model_version, + prediction_time=0, + timestamp=datetime.now().isoformat(), + metadata={"error": str(result)} + )) + else: + responses.append(result) + + return responses + + def _predict_sync(self, request: PredictionRequest) -> PredictionResponse: + """Synchronous prediction for thread pool.""" + return asyncio.run(self.model_server.predict(request)) + + async def predict_streaming(self, request_generator): + """Stream predictions as they complete.""" + loop = asyncio.get_event_loop() + + async for request in request_generator: + prediction = await self.model_server.predict(request) + yield prediction +``` + +### 3. Model Caching and Optimization +Implement model caching and prediction optimization. + +```python +from functools import lru_cache +import hashlib +import json + +class CachedModelServer(ModelServer): + """Model server with caching capabilities.""" + + def __init__(self, *args, cache_size: int = 1000, cache_ttl: int = 3600, **kwargs): + super().__init__(*args, **kwargs) + self.cache_size = cache_size + self.cache_ttl = cache_ttl + self.cache = {} + self.cache_timestamps = {} + self.logger = logging.getLogger("CachedModelServer") + + def _generate_cache_key(self, model_name: str, model_version: str, input_data: Dict[str, Any]) -> str: + """Generate cache key.""" + cache_input = f"{model_name}_{model_version}_{json.dumps(input_data, sort_keys=True)}" + return hashlib.sha256(cache_input.encode()).hexdigest() + + def _get_from_cache(self, cache_key: str) -> Optional[Any]: + """Get prediction from cache.""" + if cache_key in self.cache: + cache_time = self.cache_timestamps[cache_key] + if time.time() - cache_time < self.cache_ttl: + self.logger.debug(f"Cache hit for key: {cache_key}") + return self.cache[cache_key] + else: + # Cache expired + del self.cache[cache_key] + del self.cache_timestamps[cache_key] + + return None + + def _set_cache(self, cache_key: str, prediction: Any): + """Set prediction in cache.""" + # Implement simple LRU cache + if len(self.cache) >= self.cache_size: + # Remove oldest entry + oldest_key = min(self.cache_timestamps.keys(), key=lambda k: self.cache_timestamps[k]) + del self.cache[oldest_key] + del self.cache_timestamps[oldest_key] + + self.cache[cache_key] = prediction + self.cache_timestamps[cache_key] = time.time() + + async def predict(self, request: PredictionRequest) -> PredictionResponse: + """Predict with caching.""" + cache_key = self._generate_cache_key(request.model_name, request.model_version, request.input_data) + + # Check cache + cached_prediction = self._get_from_cache(cache_key) + if cached_prediction is not None: + return cached_prediction + + # Make prediction + prediction = await super().predict(request) + + # Cache result + self._set_cache(cache_key, prediction) + + return prediction + + def clear_cache(self): + """Clear prediction cache.""" + self.cache.clear() + self.cache_timestamps.clear() + self.logger.info("Cache cleared") +``` + +### 4. Model Version Management +Manage multiple model versions and A/B testing. + +```python +class ModelVersionManager: + """Manage model versions and A/B testing.""" + + def __init__(self, model_server: ModelServer): + self.model_server = model_server + self.traffic_rules = {} + self.logger = logging.getLogger("ModelVersionManager") + + def set_traffic_split(self, model_name: str, version_rules: Dict[str, float]): + """Set traffic split for model versions.""" + total_percentage = sum(version_rules.values()) + if abs(total_percentage - 1.0) > 0.01: + raise ValueError(f"Traffic split must sum to 1.0, got {total_percentage}") + + self.traffic_rules[model_name] = version_rules + self.logger.info(f"Set traffic split for {model_name}: {version_rules}") + + def get_model_for_request(self, model_name: str, request_id: str) -> str: + """Determine which model version to use for request.""" + if model_name not in self.traffic_rules: + return "latest" # Default to latest + + version_rules = self.traffic_rules[model_name] + + # Use request_id hash for consistent routing + hash_value = int(hashlib.md5(request_id.encode()).hexdigest(), 16) + random_value = (hash_value % 100) / 100.0 + + cumulative = 0.0 + for version, percentage in version_rules.items(): + cumulative += percentage + if random_value <= cumulative: + return version + + return list(version_rules.keys())[-1] # Fallback to last version + + async def predict_with_routing(self, request: PredictionRequest, request_id: str) -> PredictionResponse: + """Predict with intelligent version routing.""" + # Determine version + version = self.get_model_for_request(request.model_name, request_id) + request.model_version = version + + # Make prediction + prediction = await self.model_server.predict(request) + prediction.metadata = prediction.metadata or {} + prediction.metadata["routed_version"] = version + prediction.metadata["request_id"] = request_id + + return prediction + +class ABTestFramework: + """A/B testing framework for models.""" + + def __init__(self, version_manager: ModelVersionManager): + self.version_manager = version_manager + self.metrics = defaultdict(lambda: {"predictions": 0, "errors": 0, "latencies": []}) + + async def predict_with_tracking(self, request: PredictionRequest, request_id: str) -> PredictionResponse: + """Predict with A/B test tracking.""" + prediction = await self.version_manager.predict_with_routing(request, request_id) + + # Track metrics + version = prediction.metadata.get("routed_version", "unknown") + self.metrics[version]["predictions"] += 1 + self.metrics[version]["latencies"].append(prediction.prediction_time) + + return prediction + + def get_ab_test_results(self) -> Dict[str, Any]: + """Get A/B test results.""" + results = {} + for version, data in self.metrics.items(): + latencies = data["latencies"] + results[version] = { + "predictions": data["predictions"], + "errors": data["errors"], + "avg_latency": sum(latencies) / len(latencies) if latencies else 0, + "p95_latency": sorted(latencies)[int(len(latencies) * 0.95)] if len(latencies) > 20 else 0, + "error_rate": data["errors"] / data["predictions"] if data["predictions"] > 0 else 0 + } + + return results +``` + +### 5. Monitoring and Observability +Implement comprehensive monitoring for model serving. + +```python +from prometheus_client import Counter, Histogram, Gauge, Info +import logging +from logging.handlers import RotatingFileHandler + +class ModelServingMonitor: + """Monitor model serving metrics.""" + + def __init__(self): + # Prometheus metrics + self.request_counter = Counter('model_requests_total', 'Total model requests', ['model_name', 'version', 'status']) + self.request_duration = Histogram('model_request_duration_seconds', 'Model request duration', ['model_name', 'version']) + self.model_load_counter = Counter('model_loads_total', 'Total model loads', ['model_name', 'version', 'status']) + self.active_models = Gauge('active_models', 'Number of loaded models') + self.memory_usage = Gauge('memory_usage_bytes', 'Memory usage') + self.cpu_usage = Gauge('cpu_usage_percent', 'CPU usage percent') + + # Setup logging + self._setup_logging() + + def _setup_logging(self): + """Setup detailed logging.""" + logger = logging.getLogger("ModelServing") + logger.setLevel(logging.INFO) + + # Rotating file handler + handler = RotatingFileHandler('model_serving.log', maxBytes=10*1024*1024, backupCount=5) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) +``` + +## Constraints +- **Performance**: Model loading and prediction should be optimized for low latency +- **Scalability**: Design for horizontal scaling and load balancing +- **Monitoring**: Implement comprehensive monitoring and alerting +- **Error Handling**: Robust error handling and graceful degradation +- **Security**: Implement authentication, authorization, and input validation +- **Resource Management**: Monitor and manage memory, CPU, and GPU usage + +## Expected Output +Production-ready model serving infrastructure with proper API design, monitoring, caching, and scalability for reliable ML model deployment. \ No newline at end of file diff --git a/TRAE-Skills/ai_engineering/AI_Monitoring_Observability.md b/TRAE-Skills/ai_engineering/AI_Monitoring_Observability.md new file mode 100644 index 0000000000000000000000000000000000000000..53dda684d53ddbedec98dbcf266ed7aaa202eb1d --- /dev/null +++ b/TRAE-Skills/ai_engineering/AI_Monitoring_Observability.md @@ -0,0 +1,536 @@ +# Skill: AI Monitoring and Observability + +## Purpose +To implement comprehensive monitoring, logging, and observability for AI systems in production to ensure reliability, performance, and safety. + +## When to Use +- When deploying AI models to production +- When monitoring model performance and drift +- When troubleshooting AI system issues +- When ensuring SLA compliance and user satisfaction + +## Procedure + +### 1. Model Performance Monitoring +Track key performance metrics in real-time. + +```python +import time +import logging +from datetime import datetime +from collections import defaultdict +import json + +class ModelPerformanceMonitor: + def __init__(self, model_name): + self.model_name = model_name + self.metrics = defaultdict(list) + self.logger = self._setup_logger() + + def _setup_logger(self): + """Setup logging configuration.""" + logger = logging.getLogger(f"{self.model_name}_monitor") + logger.setLevel(logging.INFO) + + handler = logging.StreamHandler() + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + handler.setFormatter(formatter) + logger.addHandler(handler) + + return logger + + def log_prediction(self, input_data, prediction, latency_ms, metadata=None): + """Log individual prediction with metadata.""" + timestamp = datetime.now().isoformat() + + log_entry = { + 'timestamp': timestamp, + 'model': self.model_name, + 'input_hash': hash(str(input_data)), + 'prediction': str(prediction), + 'latency_ms': latency_ms, + 'metadata': metadata or {} + } + + self.metrics['predictions'].append(log_entry) + self.logger.info(f"Prediction logged: {latency_ms}ms") + + return log_entry + + def log_feedback(self, prediction_id, feedback): + """Log user feedback on predictions.""" + feedback_entry = { + 'prediction_id': prediction_id, + 'feedback': feedback, + 'timestamp': datetime.now().isoformat() + } + + self.metrics['feedback'].append(feedback_entry) + self.logger.info(f"Feedback received: {feedback}") + + return feedback_entry + + def calculate_metrics(self, time_window_minutes=5): + """Calculate performance metrics over time window.""" + cutoff_time = datetime.now().timestamp() - (time_window_minutes * 60) + + recent_predictions = [ + p for p in self.metrics['predictions'] + if datetime.fromisoformat(p['timestamp']).timestamp() > cutoff_time + ] + + if not recent_predictions: + return {'error': 'No recent predictions'} + + latencies = [p['latency_ms'] for p in recent_predictions] + + metrics = { + 'total_predictions': len(recent_predictions), + 'avg_latency_ms': sum(latencies) / len(latencies), + 'p50_latency': sorted(latencies)[len(latencies) // 2], + 'p95_latency': sorted(latencies)[int(len(latencies) * 0.95)], + 'p99_latency': sorted(latencies)[int(len(latencies) * 0.99)], + 'max_latency': max(latencies), + 'min_latency': min(latencies) + } + + # Calculate accuracy if feedback available + if self.metrics['feedback']: + recent_feedback = [ + f for f in self.metrics['feedback'] + if datetime.fromisoformat(f['timestamp']).timestamp() > cutoff_time + ] + + if recent_feedback: + correct = sum(1 for f in recent_feedback if f['feedback'] == 'correct') + metrics['accuracy'] = correct / len(recent_feedback) + + return metrics + + def check_sla_compliance(self, sla_max_latency_ms=500): + """Check if SLA requirements are met.""" + metrics = self.calculate_metrics() + + if 'error' in metrics: + return {'status': 'error', 'message': metrics['error']} + + compliance = { + 'sla_max_latency_ms': sla_max_latency_ms, + 'p95_latency': metrics['p95_latency'], + 'sla_met': metrics['p95_latency'] <= sla_max_latency_ms, + 'avg_latency': metrics['avg_latency_ms'] + } + + if not compliance['sla_met']: + self.logger.warning(f"SLA violation: P95 latency {metrics['p95_latency']:.2f}ms exceeds {sla_max_latency_ms}ms") + + return compliance + +# Usage +# monitor = ModelPerformanceMonitor("sentiment_model") +# +# start_time = time.time() +# prediction = model.predict("This is great!") +# latency = (time.time() - start_time) * 1000 +# +# monitor.log_prediction("This is great!", prediction, latency) +# +# metrics = monitor.calculate_metrics() +# sla_status = monitor.check_sla_compliance(sla_max_latency_ms=500) +``` + +### 2. Data Drift Detection +Monitor and detect data distribution changes. + +```python +import numpy as np +from scipy import stats +from typing import Dict, List + +class DataDriftDetector: + def __init__(self, reference_data, significance_level=0.05): + """ + Initialize with reference (training) data. + + Args: + reference_data: Dictionary of feature names to arrays + significance_level: Threshold for detecting drift + """ + self.reference_data = reference_data + self.significance_level = significance_level + self.drift_history = [] + + def calculate_kl_divergence(self, p, q): + """Calculate Kullback-Leibler divergence.""" + # Add small epsilon to avoid division by zero + epsilon = 1e-10 + p = p + epsilon + q = q + epsilon + + return np.sum(p * np.log(p / q)) + + def detect_drift(self, new_data): + """Detect drift in new data compared to reference.""" + drift_report = { + 'timestamp': datetime.now().isoformat(), + 'features': {}, + 'overall_drift_detected': False + } + + for feature in self.reference_data.keys(): + if feature not in new_data: + continue + + ref_values = self.reference_data[feature] + new_values = new_data[feature] + + # Kolmogorov-Smirnov test + ks_statistic, ks_pvalue = stats.ks_2samp(ref_values, new_values) + + # Calculate distribution statistics + ref_mean, ref_std = np.mean(ref_values), np.std(ref_values) + new_mean, new_std = np.mean(new_values), np.std(new_values) + + # Feature drift detected if p-value < significance level + feature_drift = ks_pvalue < self.significance_level + + feature_report = { + 'drift_detected': feature_drift, + 'ks_statistic': ks_statistic, + 'ks_pvalue': ks_pvalue, + 'reference_mean': ref_mean, + 'new_mean': new_mean, + 'reference_std': ref_std, + 'new_std': new_std, + 'mean_shift': abs(new_mean - ref_mean) + } + + drift_report['features'][feature] = feature_report + + if feature_drift: + drift_report['overall_drift_detected'] = True + + self.drift_history.append(drift_report) + return drift_report + + def get_drift_summary(self, window_size=10): + """Get summary of recent drift detections.""" + recent_drift = self.drift_history[-window_size:] + + if not recent_drift: + return {'status': 'No drift history available'} + + summary = { + 'total_checks': len(recent_drift), + 'drift_detected_count': sum(1 for r in recent_drift if r['overall_drift_detected']), + 'drift_rate': sum(1 for r in recent_drift if r['overall_drift_detected']) / len(recent_drift), + 'most_drifted_features': self._get_most_drifted_features(recent_drift) + } + + return summary + + def _get_most_drifted_features(self, drift_reports): + """Identify features with most frequent drift.""" + feature_drift_counts = defaultdict(int) + + for report in drift_reports: + for feature, feature_report in report['features'].items(): + if feature_report['drift_detected']: + feature_drift_counts[feature] += 1 + + return sorted(feature_drift_counts.items(), key=lambda x: x[1], reverse=True) + +# Usage +# reference_data = { +# 'age': np.random.normal(35, 10, 1000), +# 'income': np.random.normal(50000, 15000, 1000) +# } +# +# detector = DataDriftDetector(reference_data) +# +# # Simulate new data with drift +# new_data = { +# 'age': np.random.normal(45, 10, 100), # Drifted age +# 'income': np.random.normal(52000, 15000, 100) # Similar income +# } +# +# drift_report = detector.detect_drift(new_data) +# print(drift_report) +``` + +### 3. Error Analysis and Tracking +Track and analyze model errors. + +```python +class ErrorAnalyzer: + def __init__(self): + self.errors = [] + self.error_categories = defaultdict(int) + + def log_error(self, input_data, prediction, ground_truth, error_type, metadata=None): + """Log model error with details.""" + error_entry = { + 'timestamp': datetime.now().isoformat(), + 'input_data': str(input_data), + 'prediction': prediction, + 'ground_truth': ground_truth, + 'error_type': error_type, + 'metadata': metadata or {} + } + + self.errors.append(error_entry) + self.error_categories[error_type] += 1 + + return error_entry + + def analyze_error_patterns(self): + """Analyze patterns in errors.""" + if not self.errors: + return {'status': 'No errors to analyze'} + + analysis = { + 'total_errors': len(self.errors), + 'error_types': dict(self.error_categories), + 'error_rate_by_type': {}, + 'recent_errors': self.errors[-10:] # Last 10 errors + } + + # Calculate error rates + total_errors = len(self.errors) + for error_type, count in self.error_categories.items(): + analysis['error_rate_by_type'][error_type] = count / total_errors + + return analysis + + def identify_error_clusters(self, feature_extractor=None): + """Identify clusters of similar errors.""" + if not self.errors: + return {'status': 'No errors to cluster'} + + # Extract error features + error_features = [] + for error in self.errors: + if feature_extractor: + features = feature_extractor(error['input_data']) + else: + features = hash(error['input_data']) + + error_features.append({ + 'error': error, + 'features': features + }) + + # Simple clustering by error type + clusters = defaultdict(list) + for item in error_features: + error_type = item['error']['error_type'] + clusters[error_type].append(item['error']) + + return { + 'num_clusters': len(clusters), + 'cluster_sizes': {k: len(v) for k, v in clusters.items()}, + 'clusters': dict(clusters) + } + + def generate_error_report(self): + """Generate comprehensive error report.""" + patterns = self.analyze_error_patterns() + clusters = self.identify_error_clusters() + + report = { + 'summary': { + 'total_errors': patterns['total_errors'], + 'error_types': patterns['error_types'] + }, + 'patterns': patterns, + 'clusters': clusters + } + + return report + +# Usage +# error_analyzer = ErrorAnalyzer() +# +# # Log some errors +# error_analyzer.log_error( +# input_data="Great product!", +# prediction="negative", +# ground_truth="positive", +# error_type="sentiment_misclassification" +# ) +# +# report = error_analyzer.generate_error_report() +``` + +### 4. System Health Monitoring +Monitor overall AI system health. + +```python +import psutil +import GPUtil + +class AISystemHealthMonitor: + def __init__(self): + self.health_metrics = [] + + def collect_system_metrics(self): + """Collect system resource usage.""" + metrics = { + 'timestamp': datetime.now().isoformat(), + 'cpu_percent': psutil.cpu_percent(interval=1), + 'memory_percent': psutil.virtual_memory().percent, + 'disk_usage': psutil.disk_usage('/').percent, + 'network_sent': psutil.net_io_counters().bytes_sent, + 'network_recv': psutil.net_io_counters().bytes_recv + } + + # GPU metrics if available + try: + gpus = GPUtil.getGPUs() + if gpus: + metrics['gpu_usage'] = gpus[0].load * 100 + metrics['gpu_memory'] = gpus[0].memoryUtil * 100 + except: + pass + + self.health_metrics.append(metrics) + return metrics + + def check_health_status(self, thresholds=None): + """Check if system is healthy based on thresholds.""" + if thresholds is None: + thresholds = { + 'cpu_percent': 80, + 'memory_percent': 85, + 'disk_usage': 90, + 'gpu_usage': 90 + } + + latest_metrics = self.collect_system_metrics() + health_status = { + 'status': 'healthy', + 'alerts': [], + 'metrics': latest_metrics + } + + # Check each metric against threshold + for metric, threshold in thresholds.items(): + if metric in latest_metrics: + value = latest_metrics[metric] + if value > threshold: + health_status['status'] = 'warning' + health_status['alerts'].append({ + 'metric': metric, + 'value': value, + 'threshold': threshold, + 'message': f'{metric} ({value:.1f}%) exceeds threshold ({threshold}%)' + }) + + return health_status + + def generate_health_report(self): + """Generate system health report.""" + health_status = self.check_health_status() + + report = f""" +System Health Report - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} +{'=' * 60} + +Status: {health_status['status'].upper()} + +System Metrics: +- CPU Usage: {health_status['metrics']['cpu_percent']:.1f}% +- Memory Usage: {health_status['metrics']['memory_percent']:.1f}% +- Disk Usage: {health_status['metrics']['disk_usage']:.1f}% +""" + + if 'gpu_usage' in health_status['metrics']: + report += f"- GPU Usage: {health_status['metrics']['gpu_usage']:.1f}%\n" + + if health_status['alerts']: + report += "\nAlerts:\n" + for alert in health_status['alerts']: + report += f" ⚠️ {alert['message']}\n" + else: + report += "\n✅ All systems operating within normal parameters.\n" + + return report + +# Usage +# health_monitor = AISystemHealthMonitor() +# health_report = health_monitor.generate_health_report() +# print(health_report) +``` + +### 5. Integration with Monitoring Platforms +Export metrics to monitoring platforms. + +```python +class MonitoringPlatformIntegration: + def __init__(self, platform='prometheus'): + self.platform = platform + self.metrics_buffer = [] + + def export_to_prometheus(self, metrics): + """Format metrics for Prometheus.""" + prometheus_metrics = [] + + for metric_name, value in metrics.items(): + # Convert metric name to Prometheus format + prom_name = metric_name.lower().replace(' ', '_') + prometheus_metrics.append(f"{prom_name} {value}") + + return '\n'.join(prometheus_metrics) + + def export_to_datadog(self, metrics): + """Format metrics for Datadog.""" + datadog_metrics = [] + + for metric_name, value in metrics.items(): + metric_data = { + 'metric': f'ai.{metric_name.lower().replace(" ", ".")}', + 'points': [[int(time.time()), value]], + 'type': 'gauge' + } + datadog_metrics.append(metric_data) + + return datadog_metrics + + def send_metrics(self, metrics, api_endpoint): + """Send metrics to monitoring platform.""" + if self.platform == 'prometheus': + formatted_metrics = self.export_to_prometheus(metrics) + # In real implementation, push to Prometheus Pushgateway + print(f"Sending to Prometheus:\n{formatted_metrics}") + + elif self.platform == 'datadog': + formatted_metrics = self.export_to_datadog(metrics) + # In real implementation, use Datadog API + print(f"Sending to Datadog: {formatted_metrics}") + + return {'status': 'sent', 'count': len(metrics)} + +# Usage +# integration = MonitoringPlatformIntegration('prometheus') +# +# metrics = { +# 'model_latency_ms': 145.2, +# 'prediction_count': 1000, +# 'error_rate': 0.02 +# } +# +# integration.send_metrics(metrics, 'http://pushgateway:9091') +``` + +## Constraints +- **Performance Overhead**: Monitoring should not significantly impact system performance +- **Storage Costs**: Extensive logging can generate large amounts of data +- **Privacy**: Ensure sensitive data is not logged or is properly anonymized +- **Real-time Requirements**: Balance between real-time monitoring and batch processing +- **Alert Fatigue**: Configure thresholds to avoid excessive false alarms +- **Data Retention**: Implement proper data retention policies for logs and metrics + +## Expected Output +Comprehensive monitoring and observability for AI systems with real-time performance tracking, drift detection, error analysis, and system health monitoring for reliable production deployment. diff --git a/TRAE-Skills/ai_engineering/AI_Pipeline_Automation.md b/TRAE-Skills/ai_engineering/AI_Pipeline_Automation.md new file mode 100644 index 0000000000000000000000000000000000000000..623702ee7048cf2022b0b9de8ce3fe12be6595b4 --- /dev/null +++ b/TRAE-Skills/ai_engineering/AI_Pipeline_Automation.md @@ -0,0 +1,562 @@ +# Skill: AI Pipeline Automation + +## Purpose +To create automated end-to-end machine learning pipelines that handle data ingestion, preprocessing, training, evaluation, and deployment. + +## When to Use +- When building production ML systems +- When implementing continuous training and deployment +- When scaling ML workflows +- When ensuring reproducibility in ML experiments + +## Procedure + +### 1. Pipeline Framework Setup +Create a modular pipeline framework. + +```python +from abc import ABC, abstractmethod +from typing import Any, Dict, List +import logging +from datetime import datetime + +class PipelineStep(ABC): + """Abstract base class for pipeline steps.""" + + def __init__(self, name: str): + self.name = name + self.logger = logging.getLogger(f"Pipeline.{name}") + + @abstractmethod + def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Execute the pipeline step.""" + pass + + def __call__(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Execute with logging.""" + self.logger.info(f"Starting step: {self.name}") + start_time = datetime.now() + + try: + result = self.execute(context) + duration = (datetime.now() - start_time).total_seconds() + self.logger.info(f"Completed step: {self.name} in {duration:.2f}s") + return result + except Exception as e: + self.logger.error(f"Failed step: {self.name} - {str(e)}") + raise + +class Pipeline: + """Machine learning pipeline orchestrator.""" + + def __init__(self, name: str, steps: List[PipelineStep]): + self.name = name + self.steps = steps + self.logger = logging.getLogger(f"Pipeline.{name}") + self.context = {} + + def add_step(self, step: PipelineStep): + """Add a step to the pipeline.""" + self.steps.append(step) + + def run(self, initial_context: Dict[str, Any] = None) -> Dict[str, Any]: + """Run the entire pipeline.""" + self.context = initial_context or {} + + self.logger.info(f"Starting pipeline: {self.name}") + pipeline_start = datetime.now() + + try: + for step in self.steps: + self.context = step(self.context) + + duration = (datetime.now() - pipeline_start).total_seconds() + self.logger.info(f"Pipeline {self.name} completed in {duration:.2f}s") + + return self.context + + except Exception as e: + self.logger.error(f"Pipeline {self.name} failed: {str(e)}") + raise +``` + +### 2. Data Ingestion Step +Automated data collection and loading. + +```python +import pandas as pd +from sqlalchemy import create_engine +import requests +from io import StringIO + +class DataIngestionStep(PipelineStep): + """Ingest data from various sources.""" + + def __init__(self, sources: Dict[str, Any]): + super().__init__("data_ingestion") + self.sources = sources + + def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Ingest data from configured sources.""" + data = {} + + for source_name, source_config in self.sources.items(): + source_type = source_config.get('type') + + if source_type == 'csv': + data[source_name] = pd.read_csv(source_config['path']) + + elif source_type == 'database': + engine = create_engine(source_config['connection_string']) + query = source_config['query'] + data[source_name] = pd.read_sql(query, engine) + + elif source_type == 'api': + response = requests.get(source_config['url']) + response.raise_for_status() + json_data = response.json() + data[source_name] = pd.DataFrame(json_data) + + elif source_type == 'json': + data[source_name] = pd.read_json(source_config['path']) + + self.logger.info(f"Ingested {len(data[source_name])} rows from {source_name}") + + context['raw_data'] = data + return context +``` + +### 3. Data Preprocessing Step +Automated data cleaning and feature engineering. + +```python +from sklearn.preprocessing import StandardScaler, LabelEncoder +from sklearn.model_selection import train_test_split +import numpy as np + +class DataPreprocessingStep(PipelineStep): + """Preprocess and prepare data for training.""" + + def __init__(self, preprocessing_config: Dict[str, Any]): + super().__init__("data_preprocessing") + self.config = preprocessing_config + + def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Preprocess data according to configuration.""" + raw_data = context['raw_data'] + processed_data = {} + + for data_name, df in raw_data.items(): + # Handle missing values + if self.config.get('drop_missing', False): + df = df.dropna() + elif self.config.get('fill_missing'): + df = df.fillna(self.config['fill_missing']) + + # Encode categorical variables + if self.config.get('encode_categorical', False): + categorical_cols = df.select_dtypes(include=['object']).columns + for col in categorical_cols: + le = LabelEncoder() + df[col] = le.fit_transform(df[col].astype(str)) + + # Feature scaling + if self.config.get('scale_features', False): + numeric_cols = df.select_dtypes(include=[np.number]).columns + scaler = StandardScaler() + df[numeric_cols] = scaler.fit_transform(df[numeric_cols]) + + # Store scaler for later use + context[f'{data_name}_scaler'] = scaler + + processed_data[data_name] = df + self.logger.info(f"Preprocessed {data_name}: {df.shape}") + + context['processed_data'] = processed_data + return context + +class DataSplitStep(PipelineStep): + """Split data into train, validation, and test sets.""" + + def __init__(self, target_column: str, test_size: float = 0.2, val_size: float = 0.1): + super().__init__("data_split") + self.target_column = target_column + self.test_size = test_size + self.val_size = val_size + + def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Split data for training.""" + processed_data = context['processed_data'] + + for data_name, df in processed_data.items(): + if self.target_column not in df.columns: + self.logger.warning(f"Target column {self.target_column} not in {data_name}") + continue + + X = df.drop(columns=[self.target_column]) + y = df[self.target_column] + + # First split: separate test set + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=self.test_size, random_state=42 + ) + + # Second split: separate validation set from training + val_ratio = self.val_size / (1 - self.test_size) + X_train, X_val, y_train, y_val = train_test_split( + X_train, y_train, test_size=val_ratio, random_state=42 + ) + + context[f'{data_name}_train'] = (X_train, y_train) + context[f'{data_name}_val'] = (X_val, y_val) + context[f'{data_name}_test'] = (X_test, y_test) + + self.logger.info(f"Split {data_name}: train={len(X_train)}, val={len(X_val)}, test={len(X_test)}") + + return context +``` + +### 4. Model Training Step +Automated model training with hyperparameter tuning. + +```python +from sklearn.ensemble import RandomForestClassifier +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GridSearchCV +import joblib + +class ModelTrainingStep(PipelineStep): + """Train machine learning models.""" + + def __init__(self, model_type: str, hyperparameters: Dict[str, Any] = None): + super().__init__("model_training") + self.model_type = model_type + self.hyperparameters = hyperparameters or {} + + def _get_model(self): + """Get model instance based on type.""" + models = { + 'random_forest': RandomForestClassifier, + 'logistic_regression': LogisticRegression + } + + if self.model_type not in models: + raise ValueError(f"Unknown model type: {self.model_type}") + + return models[self.model_type](**self.hyperparameters) + + def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Train the model.""" + # Find training data + train_key = None + for key in context.keys(): + if key.endswith('_train'): + train_key = key + break + + if not train_key: + raise ValueError("No training data found in context") + + X_train, y_train = context[train_key] + + # Get validation data if available + val_key = train_key.replace('_train', '_val') + X_val = y_val = None + if val_key in context: + X_val, y_val = context[val_key] + + # Create and train model + model = self._get_model() + + self.logger.info(f"Training {self.model_type} model...") + model.fit(X_train, y_train) + + # Evaluate on validation set + if X_val is not None: + val_score = model.score(X_val, y_val) + self.logger.info(f"Validation score: {val_score:.4f}") + context['val_score'] = val_score + + # Store model + context['model'] = model + context['model_type'] = self.model_type + + return context + +class HyperparameterTuningStep(PipelineStep): + """Perform hyperparameter tuning.""" + + def __init__(self, model_type: str, param_grid: Dict[str, List], cv: int = 5): + super().__init__("hyperparameter_tuning") + self.model_type = model_type + self.param_grid = param_grid + self.cv = cv + + def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Perform grid search for hyperparameters.""" + train_key = None + for key in context.keys(): + if key.endswith('_train'): + train_key = key + break + + X_train, y_train = context[train_key] + + # Get base model + if self.model_type == 'random_forest': + base_model = RandomForestClassifier() + elif self.model_type == 'logistic_regression': + base_model = LogisticRegression(max_iter=1000) + else: + raise ValueError(f"Unknown model type: {self.model_type}") + + # Perform grid search + grid_search = GridSearchCV( + base_model, + self.param_grid, + cv=self.cv, + scoring='accuracy', + n_jobs=-1, + verbose=1 + ) + + self.logger.info("Starting hyperparameter tuning...") + grid_search.fit(X_train, y_train) + + # Store results + context['model'] = grid_search.best_estimator_ + context['best_params'] = grid_search.best_params_ + context['best_score'] = grid_search.best_score_ + + self.logger.info(f"Best parameters: {grid_search.best_params_}") + self.logger.info(f"Best cross-validation score: {grid_search.best_score_:.4f}") + + return context +``` + +### 5. Model Evaluation Step +Comprehensive model evaluation and reporting. + +```python +from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score +import matplotlib.pyplot as plt +import seaborn as sns + +class ModelEvaluationStep(PipelineStep): + """Evaluate trained models.""" + + def __init__(self, save_plots: bool = True, output_dir: str = './output'): + super().__init__("model_evaluation") + self.save_plots = save_plots + self.output_dir = output_dir + + def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Evaluate model on test set.""" + model = context.get('model') + if not model: + raise ValueError("No model found in context") + + # Find test data + test_key = None + for key in context.keys(): + if key.endswith('_test'): + test_key = key + break + + if not test_key: + raise ValueError("No test data found in context") + + X_test, y_test = context[test_key] + + # Make predictions + y_pred = model.predict(X_test) + test_score = model.score(X_test, y_test) + + # Calculate metrics + report = classification_report(y_test, y_pred, output_dict=True) + conf_matrix = confusion_matrix(y_test, y_pred) + + # Store results + context['test_score'] = test_score + context['classification_report'] = report + context['confusion_matrix'] = conf_matrix + + self.logger.info(f"Test score: {test_score:.4f}") + + # Generate plots + if self.save_plots: + self._save_confusion_matrix(conf_matrix, context.get('model_type', 'model')) + self._save_classification_report(report, context.get('model_type', 'model')) + + return context + + def _save_confusion_matrix(self, conf_matrix, model_name): + """Save confusion matrix plot.""" + plt.figure(figsize=(8, 6)) + sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues') + plt.title(f'Confusion Matrix - {model_name}') + plt.ylabel('True Label') + plt.xlabel('Predicted Label') + plt.savefig(f'{self.output_dir}/{model_name}_confusion_matrix.png') + plt.close() + + def _save_classification_report(self, report, model_name): + """Save classification report.""" + with open(f'{self.output_dir}/{model_name}_report.txt', 'w') as f: + f.write(f"Classification Report - {model_name}\n") + f.write("=" * 50 + "\n\n") + + for class_name, metrics in report.items(): + if isinstance(metrics, dict): + f.write(f"Class: {class_name}\n") + for metric, value in metrics.items(): + f.write(f" {metric}: {value:.4f}\n") + f.write("\n") +``` + +### 6. Model Deployment Step +Automated model deployment and versioning. + +```python +import hashlib +import json +from datetime import datetime + +class ModelDeploymentStep(PipelineStep): + """Deploy trained models.""" + + def __init__(self, deployment_config: Dict[str, Any]): + super().__init__("model_deployment") + self.config = deployment_config + + def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Deploy model to production.""" + model = context.get('model') + if not model: + raise ValueError("No model found in context") + + # Generate model version + model_version = self._generate_version(context) + + # Save model + model_path = f"{self.config['model_dir']}/model_{model_version}.joblib" + joblib.dump(model, model_path) + + # Save metadata + metadata = { + 'version': model_version, + 'model_type': context.get('model_type', 'unknown'), + 'test_score': context.get('test_score', 0), + 'best_params': context.get('best_params', {}), + 'timestamp': datetime.now().isoformat(), + 'model_path': model_path + } + + metadata_path = f"{self.config['model_dir']}/model_{model_version}_metadata.json" + with open(metadata_path, 'w') as f: + json.dump(metadata, f, indent=2) + + # Update model registry + self._update_model_registry(metadata) + + context['deployed_model'] = metadata + self.logger.info(f"Model deployed with version: {model_version}") + + return context + + def _generate_version(self, context: Dict[str, Any]) -> str: + """Generate unique model version.""" + data_string = f"{context.get('model_type')}{context.get('test_score')}{datetime.now().timestamp()}" + return hashlib.md5(data_string.encode()).hexdigest()[:8] + + def _update_model_registry(self, metadata: Dict[str, Any]): + """Update model registry with new deployment.""" + registry_path = f"{self.config['model_dir']}/model_registry.json" + + # Load existing registry + try: + with open(registry_path, 'r') as f: + registry = json.load(f) + except FileNotFoundError: + registry = {'models': [], 'current': None} + + # Add new model + registry['models'].append(metadata) + registry['current'] = metadata['version'] + + # Save updated registry + with open(registry_path, 'w') as f: + json.dump(registry, f, indent=2) +``` + +### 7. Complete Pipeline Example +Assemble and run the complete pipeline. + +```python +def create_ml_pipeline() -> Pipeline: + """Create a complete ML pipeline.""" + + # Define configuration + sources = { + 'training_data': { + 'type': 'csv', + 'path': './data/training_data.csv' + } + } + + preprocessing_config = { + 'drop_missing': True, + 'encode_categorical': True, + 'scale_features': True + } + + deployment_config = { + 'model_dir': './models' + } + + # Create pipeline steps + steps = [ + DataIngestionStep(sources), + DataPreprocessingStep(preprocessing_config), + DataSplitStep(target_column='target', test_size=0.2, val_size=0.1), + HyperparameterTuningStep( + model_type='random_forest', + param_grid={ + 'n_estimators': [50, 100, 200], + 'max_depth': [5, 10, 15], + 'min_samples_split': [2, 5, 10] + }, + cv=5 + ), + ModelEvaluationStep(save_plots=True, output_dir='./output'), + ModelDeploymentStep(deployment_config) + ] + + return Pipeline(name="ml_training_pipeline", steps=steps) + +# Usage +if __name__ == "__main__": + # Setup logging + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + + # Create and run pipeline + pipeline = create_ml_pipeline() + result = pipeline.run() + + print("Pipeline completed successfully!") + print(f"Test score: {result.get('test_score', 'N/A')}") +``` + +## Constraints +- **Data Quality**: Garbage in, garbage out - ensure data quality +- **Pipeline Complexity**: Balance between automation and flexibility +- **Resource Management**: Monitor computational resources during training +- **Version Control**: Track all pipeline steps and configurations +- **Error Handling**: Implement robust error handling and recovery +- **Scalability**: Design for data and computational scaling + +## Expected Output +Automated, reproducible ML pipelines that handle data ingestion, preprocessing, training, evaluation, and deployment with proper monitoring and version control. diff --git a/TRAE-Skills/ai_engineering/AI_Safety_Ethics.md b/TRAE-Skills/ai_engineering/AI_Safety_Ethics.md new file mode 100644 index 0000000000000000000000000000000000000000..159f9cdff558257e925f672d46a787ea20fd3aba --- /dev/null +++ b/TRAE-Skills/ai_engineering/AI_Safety_Ethics.md @@ -0,0 +1,379 @@ +# Skill: AI Safety and Ethics + +## Purpose +To implement AI systems that are safe, fair, transparent, and aligned with human values while minimizing potential harms and biases. + +## When to Use +- When deploying AI systems that affect people's lives +- When working with sensitive data or protected characteristics +- When implementing automated decision-making systems +- When ensuring regulatory compliance (GDPR, AI Act, etc.) + +## Procedure + +### 1. Bias Detection and Mitigation +Identify and reduce biases in AI systems. + +```python +import pandas as pd +from sklearn.metrics import confusion_matrix +from aif360.datasets import BinaryLabelDataset +from aif360.metrics import BinaryLabelDatasetMetric +from aif360.algorithms.preprocessing import Reweighing + +def detect_bias(df, protected_attribute, label, privileged_groups, unprivileged_groups): + """Detect bias in dataset.""" + # Convert to AIF360 dataset + dataset = BinaryLabelDataset( + df=df, + label_names=[label], + protected_attribute_names=[protected_attribute], + favorable_label=1, + unfavorable_label=0 + ) + + # Calculate fairness metrics + metric = BinaryLabelDatasetMetric( + dataset, + unprivileged_groups=unprivileged_groups, + privileged_groups=privileged_groups + ) + + return { + 'disparate_impact': metric.disparate_impact(), + 'statistical_parity_difference': metric.statistical_parity_difference() + } + +def mitigate_bias(df, protected_attribute, label, privileged_groups, unprivileged_groups): + """Apply bias mitigation techniques.""" + # Original dataset + dataset = BinaryLabelDataset( + df=df, + label_names=[label], + protected_attribute_names=[protected_attribute], + favorable_label=1, + unfavorable_label=0 + ) + + # Apply reweighing + reweigher = Reweighing( + unprivileged_groups=unprivileged_groups, + privileged_groups=privileged_groups + ) + + dataset_transformed = reweigher.fit_transform(dataset) + + return dataset_transformed.convert_to_dataframe()[0] + +# Example usage +# df = pd.read_csv('loan_applications.csv') +# protected_attribute = 'gender' +# label = 'loan_approved' +# +# bias_metrics = detect_bias(df, protected_attribute, label, +# privileged_groups=[{'gender': 1}], +# unprivileged_groups=[{'gender': 0}]) +# +# print(f"Disparate Impact: {bias_metrics['disparate_impact']}") +# +# # Mitigate bias +# fair_df = mitigate_bias(df, protected_attribute, label, +# privileged_groups=[{'gender': 1}], +# unprivileged_groups=[{'gender': 0}]) +``` + +### 2. Content Moderation and Safety +Implement safety filters for AI content. + +```python +import openai + +class SafeContentGenerator: + def __init__(self): + self.client = openai.OpenAI() + self.forbidden_categories = [ + "hate speech", + "violence", + "self-harm", + "sexual content", + "illegal activities" + ] + + def check_safety(self, content): + """Check if content is safe.""" + moderation_response = self.client.moderations.create(input=content) + result = moderation_response.results[0] + + return { + 'flagged': result.flagged, + 'categories': result.categories, + 'category_scores': result.category_scores + } + + def generate_safe_content(self, prompt, max_retries=3): + """Generate content with safety checks.""" + for attempt in range(max_retries): + response = self.client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + temperature=0.7 + ) + + content = response.choices[0].message.content + safety_check = self.check_safety(content) + + if not safety_check['flagged']: + return content + + print(f"Attempt {attempt + 1}: Content flagged as unsafe") + prompt = f"Generate content that is completely safe and appropriate: {prompt}" + + raise Exception("Failed to generate safe content after multiple attempts") + +# Usage +generator = SafeContentGenerator() +safe_content = generator.generate_safe_content("Write a story about teamwork") +``` + +### 3. Transparency and Explainability +Implement explainability for AI decisions. + +```python +import shap +from sklearn.ensemble import RandomForestClassifier + +class ExplainableAI: + def __init__(self, model, feature_names): + self.model = model + self.feature_names = feature_names + self.explainer = None + + def fit_explainer(self, X_train): + """Fit SHAP explainer.""" + self.explainer = shap.Explainer(self.model, X_train) + + def explain_prediction(self, instance): + """Explain individual prediction.""" + shap_values = self.explainer(instance) + + # Get feature importance + feature_importance = list(zip( + self.feature_names, + shap_values.values[0] + )) + + # Sort by absolute importance + feature_importance.sort(key=lambda x: abs(x[1]), reverse=True) + + return feature_importance[:10] # Top 10 features + + def generate_explanation_text(self, instance, prediction, feature_importance): + """Generate human-readable explanation.""" + explanation = f"Prediction: {prediction}\n\n" + explanation += "Key factors influencing this decision:\n" + + for feature, importance in feature_importance: + direction = "increases" if importance > 0 else "decreases" + explanation += f"- {feature}: {direction} likelihood (impact: {abs(importance):.3f})\n" + + return explanation + +# Example usage +# model = RandomForestClassifier() +# model.fit(X_train, y_train) +# +# explainable_ai = ExplainableAI(model, feature_names) +# explainable_ai.fit_explainer(X_train) +# +# instance = X_test[0] +# prediction = model.predict([instance])[0] +# feature_importance = explainable_ai.explain_prediction(instance) +# +# explanation = explainable_ai.generate_explanation_text( +# instance, prediction, feature_importance +# ) +# print(explanation) +``` + +### 4. Privacy-Preserving AI +Implement privacy protection in AI systems. + +```python +import numpy as np +from sklearn.preprocessing import StandardScaler + +class PrivacyPreservingML: + def __init__(self, epsilon=1.0): + self.epsilon = epsilon + self.scaler = StandardScaler() + + def add_laplace_noise(self, data, sensitivity): + """Add Laplace noise for differential privacy.""" + scale = sensitivity / self.epsilon + noise = np.random.laplace(0, scale, data.shape) + return data + noise + + def private_aggregation(self, data): + """Perform differentially private aggregation.""" + sensitivity = 1.0 # Depends on the query + noisy_mean = self.add_laplace_noise(data, sensitivity) + return np.mean(noisy_mean) + + def anonymize_data(self, df, sensitive_columns): + """Anonymize sensitive data.""" + df_anon = df.copy() + + for column in sensitive_columns: + # Generalize or remove sensitive information + if df[column].dtype == 'object': + # Categorical: use frequency encoding + freq = df[column].value_counts(normalize=True) + df_anon[column] = df[column].map(freq) + else: + # Numerical: add noise + df_anon[column] = self.add_laplace_noise( + df[column].values, + sensitivity=df[column].max() + ) + + return df_anon + + def federated_learning_step(self, local_models, global_model): + """Simulate federated learning update.""" + # Aggregate local model updates + averaged_weights = [] + + for weights_list in zip(*[model.get_weights() for model in local_models]): + averaged_weights.append(np.mean(weights_list, axis=0)) + + # Update global model + global_model.set_weights(averaged_weights) + return global_model + +# Usage +# privacy_ml = PrivacyPreservingML(epsilon=0.5) +# +# # Anonymize data +# sensitive_data = df[['age', 'income', 'zip_code']] +# df_anon = privacy_ml.anonymize_data(df, ['age', 'income']) +``` + +### 5. Ethical Review Framework +Implement ethical review for AI systems. + +```python +class EthicalReviewer: + def __init__(self): + self.checklist = { + 'fairness': [ + 'Have we tested for bias across demographic groups?', + 'Are the training datasets representative?', + 'Have we implemented fairness metrics?' + ], + 'transparency': [ + 'Can we explain individual decisions?', + 'Are users informed about AI involvement?', + 'Is the system\'s limitations documented?' + ], + 'accountability': [ + 'Is there a human in the loop?', + 'Can decisions be appealed?', + 'Are logs maintained for audit?' + ], + 'safety': [ + 'Have we implemented safety guards?', + 'Is there content moderation?', + 'Are there fail-safe mechanisms?' + ], + 'privacy': [ + 'Is user data protected?', + 'Have we obtained proper consent?', + 'Is data minimization applied?' + ] + } + + def review_system(self, responses): + """Review AI system against ethical checklist.""" + results = {} + + for category, questions in self.checklist.items(): + category_score = 0 + category_responses = [] + + for i, question in enumerate(questions): + response = responses.get(f"{category}_{i}", "not_answered") + + if response.lower() in ['yes', 'implemented', 'yes_implemented']: + category_score += 1 + status = "✓" + elif response.lower() == 'partially': + category_score += 0.5 + status = "~" + else: + status = "✗" + + category_responses.append({ + 'question': question, + 'response': response, + 'status': status + }) + + results[category] = { + 'score': category_score / len(questions), + 'responses': category_responses + } + + return results + + def generate_report(self, review_results): + """Generate ethical review report.""" + report = "AI System Ethical Review Report\n" + report += "=" * 50 + "\n\n" + + for category, data in review_results.items(): + score_percent = data['score'] * 100 + report += f"{category.upper()}: {score_percent:.0f}%\n" + + for response in data['responses']: + report += f" {response['status']} {response['question']}\n" + report += f" Response: {response['response']}\n" + + report += "\n" + + overall_score = np.mean([data['score'] for data in review_results.values()]) + report += f"\nOVERALL ETHICAL SCORE: {overall_score * 100:.0f}%\n" + + if overall_score >= 0.8: + report += "Status: PASS - System meets ethical standards\n" + elif overall_score >= 0.6: + report += "Status: CONDITIONAL - Address identified concerns\n" + else: + report += "Status: FAIL - Significant ethical concerns\n" + + return report + +# Usage +# reviewer = EthicalReviewer() +# +# responses = { +# 'fairness_0': 'yes', +# 'fairness_1': 'partially', +# 'transparency_0': 'yes', +# # ... more responses +# } +# +# results = reviewer.review_system(responses) +# report = reviewer.generate_report(results) +# print(report) +``` + +## Constraints +- **Legal Compliance**: Ensure compliance with relevant laws and regulations +- **Context Dependency**: Ethical considerations vary by application and culture +- **Trade-offs**: Balance between competing ethical principles may be necessary +- **Continuous Monitoring**: Ethical behavior requires ongoing monitoring and updates +- **Stakeholder Involvement**: Include diverse stakeholders in ethical assessments +- **Transparency Limits**: Some models are inherently difficult to explain + +## Expected Output +AI systems that are safe, fair, transparent, and aligned with ethical principles, with proper documentation and monitoring for responsible deployment. diff --git a/TRAE-Skills/ai_engineering/AI_Testing_Evaluation.md b/TRAE-Skills/ai_engineering/AI_Testing_Evaluation.md new file mode 100644 index 0000000000000000000000000000000000000000..38abd805f6c90d04bf94dd27e10c6ece8f5afacf --- /dev/null +++ b/TRAE-Skills/ai_engineering/AI_Testing_Evaluation.md @@ -0,0 +1,400 @@ +# Skill: AI Testing and Evaluation + +## Purpose +To systematically evaluate AI model performance, reliability, and safety using comprehensive testing methodologies and metrics. + +## When to Use +- When validating AI models before deployment +- When comparing different models or approaches +- When monitoring model performance in production +- When ensuring AI system reliability and safety + +## Procedure + +### 1. Model Performance Metrics +Calculate comprehensive performance metrics. + +```python +import numpy as np +from sklearn.metrics import ( + accuracy_score, precision_score, recall_score, f1_score, + confusion_matrix, roc_auc_score, classification_report +) + +class ModelEvaluator: + def __init__(self): + self.metrics = {} + + def evaluate_classification(self, y_true, y_pred, y_prob=None): + """Evaluate classification model performance.""" + self.metrics['accuracy'] = accuracy_score(y_true, y_pred) + self.metrics['precision'] = precision_score(y_true, y_pred, average='weighted') + self.metrics['recall'] = recall_score(y_true, y_pred, average='weighted') + self.metrics['f1_score'] = f1_score(y_true, y_pred, average='weighted') + + if y_prob is not None: + self.metrics['roc_auc'] = roc_auc_score(y_true, y_prob, multi_class='ovr') + + # Confusion matrix + self.metrics['confusion_matrix'] = confusion_matrix(y_true, y_pred) + + return self.metrics + + def evaluate_regression(self, y_true, y_pred): + """Evaluate regression model performance.""" + from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score + + self.metrics['mse'] = mean_squared_error(y_true, y_pred) + self.metrics['rmse'] = np.sqrt(self.metrics['mse']) + self.metrics['mae'] = mean_absolute_error(y_true, y_pred) + self.metrics['r2_score'] = r2_score(y_true, y_pred) + + return self.metrics + + def generate_report(self): + """Generate evaluation report.""" + report = "Model Evaluation Report\n" + report += "=" * 40 + "\n" + + for metric, value in self.metrics.items(): + if metric != 'confusion_matrix': + if isinstance(value, float): + report += f"{metric}: {value:.4f}\n" + else: + report += f"{metric}: {value}\n" + + return report + +# Usage +# evaluator = ModelEvaluator() +# +# y_true = [0, 1, 2, 1, 0] +# y_pred = [0, 2, 2, 1, 0] +# +# metrics = evaluator.evaluate_classification(y_true, y_pred) +# print(evaluator.generate_report()) +``` + +### 2. LLM Quality Evaluation +Evaluate language model outputs. + +```python +from openai import OpenAI +import json + +class LLMEvaluator: + def __init__(self, model="gpt-4"): + self.client = OpenAI() + self.model = model + + def evaluate_relevance(self, question, answer, reference_answer=None): + """Evaluate answer relevance.""" + prompt = f""" + Question: {question} + Answer: {answer} + + Rate the relevance of this answer to the question on a scale of 1-10. + Consider: Does it directly address the question? Is it comprehensive? + + Provide rating as JSON: {{"relevance_score": , "reasoning": ""}} + """ + + response = self.client.chat.completions.create( + model=self.model, + messages=[{"role": "user", "content": prompt}], + temperature=0 + ) + + try: + result = json.loads(response.choices[0].message.content) + return result + except: + return {"relevance_score": 0, "reasoning": "Failed to parse evaluation"} + + def evaluate_accuracy(self, question, answer, ground_truth): + """Evaluate factual accuracy.""" + prompt = f""" + Question: {question} + Generated Answer: {answer} + Ground Truth: {ground_truth} + + Compare the generated answer with the ground truth. + Rate factual accuracy on a scale of 1-10. + + Provide rating as JSON: {{"accuracy_score": , "errors": []}} + """ + + response = self.client.chat.completions.create( + model=self.model, + messages=[{"role": "user", "content": prompt}], + temperature=0 + ) + + try: + result = json.loads(response.choices[0].message.content) + return result + except: + return {"accuracy_score": 0, "errors": ["Failed to parse evaluation"]} + + def evaluate_toxicity(self, text): + """Check for toxic content.""" + moderation_response = self.client.moderations.create(input=text) + result = moderation_response.results[0] + + return { + 'flagged': result.flagged, + 'categories': result.categories, + 'category_scores': result.category_scores + } + + def batch_evaluate(self, test_cases): + """Evaluate multiple test cases.""" + results = [] + + for case in test_cases: + result = { + 'question': case['question'], + 'answer': case['answer'] + } + + # Run evaluations + result['relevance'] = self.evaluate_relevance( + case['question'], + case['answer'] + ) + + if 'ground_truth' in case: + result['accuracy'] = self.evaluate_accuracy( + case['question'], + case['answer'], + case['ground_truth'] + ) + + result['toxicity'] = self.evaluate_toxicity(case['answer']) + + results.append(result) + + return results + + def calculate_aggregate_metrics(self, evaluation_results): + """Calculate aggregate metrics across all test cases.""" + metrics = { + 'avg_relevance': np.mean([r['relevance']['relevance_score'] for r in evaluation_results]), + 'flagged_count': sum([1 for r in evaluation_results if r['toxicity']['flagged']]), + 'total_cases': len(evaluation_results) + } + + if 'accuracy' in evaluation_results[0]: + metrics['avg_accuracy'] = np.mean([r['accuracy']['accuracy_score'] for r in evaluation_results]) + + return metrics + +# Usage +# evaluator = LLMEvaluator() +# +# test_cases = [ +# { +# 'question': 'What is machine learning?', +# 'answer': 'Machine learning is a subset of AI that enables systems to learn from data.', +# 'ground_truth': 'Machine learning involves training algorithms to make predictions or decisions based on data.' +# } +# ] +# +# results = evaluator.batch_evaluate(test_cases) +# aggregate = evaluator.calculate_aggregate_metrics(results) +# print(aggregate) +``` + +### 3. Robustness Testing +Test model robustness against adversarial inputs. + +```python +class RobustnessTester: + def __init__(self, model): + self.model = model + + def test_typos(self, text, num_variations=5): + """Test model with typographical variations.""" + variations = [] + + # Common typos + common_typos = { + 'the': 'teh', + 'and': 'adn', + 'is': 'si', + 'to': 'ot', + 'of': 'fo' + } + + for _ in range(num_variations): + variation = text + for correct, typo in common_typos.items(): + variation = variation.replace(correct, typo) + variations.append(variation) + + return variations + + def test_adversarial_examples(self, texts, labels, attack_method='textbugger'): + """Test against adversarial attacks.""" + # Simplified adversarial example generation + adversarial_examples = [] + + for text, label in zip(texts, labels): + # Add slight perturbations + words = text.split() + if len(words) > 0: + # Duplicate a word + words.append(words[0]) + adversarial = ' '.join(words) + adversarial_examples.append((adversarial, label)) + + return adversarial_examples + + def test_out_of_distribution(self, texts): + """Test with out-of-distribution inputs.""" + ood_cases = [ + # Very short inputs + "Hi", + "A", + "", + + # Very long inputs + "word " * 1000, + + # Special characters + "!@#$%^&*()", + + # Mixed languages + "Hello 你好 Bonjour", + + # Malformed inputs + "...test...", + "123456789" + ] + + return ood_cases + + def evaluate_robustness(self, test_function): + """Evaluate model robustness.""" + test_cases = { + 'typos': self.test_typos("What is the meaning of life?"), + 'ood': self.test_out_of_distribution([]), + 'adversarial': self.test_adversarial_examples(["Hello world"], [1]) + } + + results = {} + + for category, cases in test_cases.items(): + results[category] = [] + + for case in cases: + try: + response = test_function(case) + results[category].append({ + 'input': case, + 'status': 'success', + 'response': response + }) + except Exception as e: + results[category].append({ + 'input': case, + 'status': 'failed', + 'error': str(e) + }) + + return results + +# Usage +# def mock_llm(text): +# return f"Response to: {text[:50]}" +# +# tester = RobustnessTester(model=None) +# robustness_results = tester.evaluate_robustness(mock_llm) +``` + +### 4. A/B Testing Framework +Implement A/B testing for model comparison. + +```python +from scipy import stats + +class ABTestFramework: + def __init__(self): + self.results = {'A': [], 'B': []} + + def add_result(self, group, metric_value): + """Add a result for a group.""" + if group in ['A', 'B']: + self.results[group].append(metric_value) + + def calculate_significance(self, metric='accuracy'): + """Calculate statistical significance.""" + group_a = self.results['A'] + group_b = self.results['B'] + + # T-test + t_statistic, p_value = stats.ttest_ind(group_a, group_b) + + # Effect size (Cohen's d) + pooled_std = np.sqrt((np.std(group_a)**2 + np.std(group_b)**2) / 2) + cohens_d = (np.mean(group_a) - np.mean(group_b)) / pooled_std + + return { + 'group_a_mean': np.mean(group_a), + 'group_b_mean': np.mean(group_b), + 't_statistic': t_statistic, + 'p_value': p_value, + 'significant': p_value < 0.05, + 'cohens_d': cohens_d + } + + def generate_report(self): + """Generate A/B test report.""" + stats = self.calculate_significance() + + report = f""" +A/B Test Results Report +{'=' * 50} + +Group A: n={len(self.results['A'])}, mean={stats['group_a_mean']:.4f} +Group B: n={len(self.results['B'])}, mean={stats['group_b_mean']:.4f} + +Statistical Analysis: +- t-statistic: {stats['t_statistic']:.4f} +- p-value: {stats['p_value']:.4f} +- Significant: {'Yes' if stats['significant'] else 'No'} +- Effect size (Cohen's d): {stats['cohens_d']:.4f} + +Conclusion: +""" + if stats['significant']: + if stats['group_a_mean'] > stats['group_b_mean']: + report += "Group A performs significantly better than Group B." + else: + report += "Group B performs significantly better than Group A." + else: + report += "No significant difference found between groups." + + return report + +# Usage +# ab_test = ABTestFramework() +# +# # Add results (in practice, these come from user interactions) +# for _ in range(100): +# ab_test.add_result('A', np.random.normal(0.7, 0.1)) +# ab_test.add_result('B', np.random.normal(0.75, 0.1)) +# +# print(ab_test.generate_report()) +``` + +## Constraints +- **Ground Truth**: High-quality ground truth data is essential for accurate evaluation +- **Bias in Evaluation**: Evaluation metrics themselves may contain biases +- **Context Dependency**: Performance may vary significantly across different contexts +- **Cost**: Comprehensive evaluation can be expensive, especially for LLMs +- **Subjectivity**: Some metrics (like quality) are inherently subjective +- **Dynamic Performance**: Model performance may degrade over time + +## Expected Output +Comprehensive evaluation of AI systems with detailed metrics, robustness testing results, and statistical analysis to ensure reliable and safe deployment. diff --git a/TRAE-Skills/ai_engineering/Chain_of_Thought_Prompting.md b/TRAE-Skills/ai_engineering/Chain_of_Thought_Prompting.md new file mode 100644 index 0000000000000000000000000000000000000000..1ad0a8f50373af09fddefd78da73d64a83221239 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Chain_of_Thought_Prompting.md @@ -0,0 +1,169 @@ +# Skill: Chain of Thought Prompting + +## Purpose +To improve LLM reasoning performance by encouraging models to show their work and think through problems step-by-step before arriving at an answer. + +## When to Use +- When solving complex math or logic problems +- When reasoning through multi-step questions +- When you need to verify the model's thinking process +- When working with problems requiring inference + +## Procedure + +### 1. Basic Chain of Thought +Structure your prompt to encourage step-by-step reasoning. + +```python +from openai import OpenAI + +client = OpenAI() + +prompt = """ +Question: If a store sells apples for $2 each and oranges for $3 each, +and you buy 5 apples and 3 oranges, how much do you spend? + +Let's think step by step: +1. Calculate the cost of apples: 5 apples × $2 = $10 +2. Calculate the cost of oranges: 3 oranges × $3 = $9 +3. Add both amounts: $10 + $9 = $19 + +Answer: $19 + +Question: A train travels 120 miles in 2 hours. If it maintains the same speed, +how far will it travel in 5 hours? + +Let's think step by step: +""" + +response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}] +) +print(response.choices[0].message.content) +``` + +### 2. Zero-Shot Chain of Thought +Simply add "Let's think step by step" to your prompt. + +```python +def zero_shot_cot(question): + prompt = f"""Question: {question} + +Let's think step by step:""" + + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}] + ) + return response.choices[0].message.content + +# Example +question = "If 3 workers can build a wall in 4 days, how many days will 6 workers need?" +print(zero_shot_cot(question)) +``` + +### 3. Few-Shot Chain of Thought +Provide examples of the reasoning process. + +```python +few_shot_prompt = """ +Q: Roger has 5 tennis balls. He buys 2 more cans of 3 tennis balls each. +How many tennis balls does he have now? +A: Roger started with 5 balls. 2 cans of 3 tennis balls each is 6 tennis balls. +5 + 6 = 11. The answer is 11. + +Q: A restaurant had 23 apples. If they used 20 to make lunch and bought 6 more, +how many apples do they have? +A: They had 23 apples and used 20, so 23 - 20 = 3. Then they bought 6 more, +so 3 + 6 = 9. The answer is 9. + +Q: If a car travels 60 miles per hour for 3 hours, how far does it travel? +A:""" + +response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": few_shot_prompt}] +) +print(response.choices[0].message.content) +``` + +### 4. Self-Consistency with Chain of Thought +Generate multiple reasoning paths and take the majority answer. + +```python +import numpy as np + +def self_consistent_cot(question, num_samples=5): + responses = [] + for _ in range(num_samples): + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": f"Question: {question}\nLet's think step by step:"}], + temperature=0.7 # Add randomness for diverse paths + ) + responses.append(response.choices[0].message.content) + + # Parse final answers from responses + answers = [r.split("Answer:")[-1].strip() if "Answer:" in r else r for r in responses] + + # Return most common answer + from collections import Counter + most_common = Counter(answers).most_common(1)[0][0] + return most_common + +# Example +question = "A bat and ball cost $1.10 in total. The bat costs $1.00 more than the ball. How much does the ball cost?" +answer = self_consistent_cot(question) +print(f"Final answer: {answer}") +``` + +### 5. Structured Chain of Thought +Use specific reasoning templates for different problem types. + +```python +math_template = """ +Solve this math problem systematically: + +1. Identify the given information +2. Identify what needs to be found +3. Determine the appropriate formula or approach +4. Execute the calculation +5. Verify the answer + +Question: {question} + +Answer:""" + +coding_template = """ +Debug this code step by step: + +1. Understand what the code should do +2. Trace through the execution line by line +3. Identify where the behavior differs from expectations +4. Propose and test fixes + +Code: {code} + +Issue: {issue} + +Fix:""" + +def structured_cot(template, **kwargs): + prompt = template.format(**kwargs) + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}] + ) + return response.choices[0].message.content +``` + +## Constraints +- **Token Usage**: Chain of thought increases token consumption significantly +- **Latency**: More tokens = longer response times +- **Model Selection**: Works best with larger models (GPT-4, Claude) +- **Temperature**: Use lower temperature (0-0.3) for more consistent reasoning +- **Verification**: Always verify the reasoning steps, especially for critical applications + +## Expected Output +Improved reasoning performance on complex problems that require multi-step thinking, with transparent thought processes that can be verified and debugged. diff --git a/TRAE-Skills/ai_engineering/Computer_Vision_Object_Detection.md b/TRAE-Skills/ai_engineering/Computer_Vision_Object_Detection.md new file mode 100644 index 0000000000000000000000000000000000000000..8c2ea70f27c482614e0da0b08b8167925e633fa0 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Computer_Vision_Object_Detection.md @@ -0,0 +1,95 @@ +# Skill: Computer Vision - Object Detection + +## Purpose +To locate and identify multiple objects within an image or video stream by drawing bounding boxes and assigning class labels with confidence scores. + +## When to Use +- When counting objects (e.g., people in a crowd, cars on a road) +- When building autonomous navigation systems or robotics +- When performing automated quality inspection in manufacturing +- When extracting specific elements from documents (e.g., tables, signatures) + +## Procedure + +### 1. Choose the Architecture +Select an architecture based on the trade-off between speed and accuracy: +- **YOLO (You Only Look Once)**: Best for real-time inference (YOLOv8, YOLOv10). Very fast and highly accurate. +- **Faster R-CNN**: Slower but highly accurate, especially for small objects. Good for medical imaging. +- **DETR (DEtection TRansformer)**: Transformer-based approach that eliminates the need for non-maximum suppression (NMS) and anchor boxes. + +### 2. Dataset Preparation +Ensure the dataset is properly formatted. The most common formats are: +- **COCO JSON**: A single JSON file containing images, annotations (bounding boxes, polygons), and categories. +- **YOLO TXT**: One text file per image containing `class_id x_center y_center width height` (normalized between 0 and 1). + +**Data Augmentation**: +Apply augmentations to improve robustness using libraries like `albumentations`: +```python +import albumentations as A + +transform = A.Compose([ + A.HorizontalFlip(p=0.5), + A.RandomBrightnessContrast(p=0.2), + A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=45, p=0.2), +], bbox_params=A.BboxParams(format='yolo', label_fields=['class_labels'])) +``` + +### 3. YOLOv8 Implementation Example +Using the Ultralytics library for training and inference. + +**Installation**: +```bash +pip install ultralytics +``` + +**Training**: +Create a `data.yaml` file defining the dataset paths and classes: +```yaml +train: ../train/images +val: ../valid/images + +nc: 3 # number of classes +names: ['car', 'pedestrian', 'traffic_light'] +``` + +```python +from ultralytics import YOLO + +# Load a pre-trained model (recommended for transfer learning) +model = YOLO('yolov8n.pt') # 'n' for nano, 's' for small, 'm' for medium, etc. + +# Train the model on your custom dataset +results = model.train(data='data.yaml', epochs=100, imgsz=640, batch=16, device=0) +``` + +**Inference**: +```python +from ultralytics import YOLO +import cv2 + +# Load the fine-tuned model +model = YOLO('runs/detect/train/weights/best.pt') + +# Perform inference on an image +results = model('test_image.jpg') + +# View results +for result in results: + boxes = result.boxes # Bounding boxes object + for box in boxes: + # Extract coordinates, confidence, and class id + x1, y1, x2, y2 = box.xyxy[0] + conf = box.conf[0] + cls_id = int(box.cls[0]) + print(f"Class: {model.names[cls_id]}, Confidence: {conf:.2f}, Box: [{x1}, {y1}, {x2}, {y2}]") +``` + +### 4. Evaluation Metrics +Understand the standard metrics for object detection: +- **IoU (Intersection over Union)**: Measures the overlap between the predicted bounding box and the ground truth. +- **mAP (Mean Average Precision)**: The primary metric. Often calculated at different IoU thresholds (e.g., mAP@0.5, mAP@0.5:0.95). + +## Best Practices +- Ensure a balanced dataset across all classes to prevent the model from ignoring rare objects. +- Pay attention to image size (`imgsz`). Larger sizes detect smaller objects better but require more memory and slow down inference. +- Utilize pre-trained weights (Transfer Learning) instead of training from scratch whenever possible. \ No newline at end of file diff --git a/TRAE-Skills/ai_engineering/Data_Drift_Detection.md b/TRAE-Skills/ai_engineering/Data_Drift_Detection.md new file mode 100644 index 0000000000000000000000000000000000000000..7defd394d4067984616811591024af3acbb67cb4 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Data_Drift_Detection.md @@ -0,0 +1,84 @@ +# Skill: Data Drift Detection + +## Purpose +To monitor production machine learning models for performance degradation caused by changes in the statistical properties of the input data (data drift) or the relationship between features and the target variable (concept drift) over time. + +## When to Use +- When deploying models to production environments where data changes frequently (e.g., e-commerce, fraud detection, recommendation systems) +- When establishing a continuous training pipeline (MLOps) +- When debugging silent failures in model accuracy without corresponding system errors + +## Procedure + +### 1. Identify Drift Types +Distinguish between the types of drift occurring in the data: +- **Covariate Shift (Data Drift)**: The distribution of input features ($P(X)$) changes, but the relationship with the target remains the same. +- **Prior Probability Shift**: The distribution of the target variable ($P(Y)$) changes. +- **Concept Drift**: The relationship between features and the target ($P(Y|X)$) changes (e.g., a new type of fraud emerges). + +### 2. Choose Detection Methods +Select appropriate statistical tests to compare a reference dataset (usually the training data) with the current production data: +- **Numerical Features**: Kolmogorov-Smirnov (K-S) test, Wasserstein distance, Population Stability Index (PSI). +- **Categorical Features**: Chi-Square test, Jensen-Shannon distance. +- **Multivariate**: Domain Classifier (train a model to distinguish between reference and current data). + +### 3. Implementation Example (Evidently AI) +Evidently is an open-source library for monitoring ML models. + +**Installation**: +```bash +pip install evidently +``` + +**Generate a Data Drift Report**: +```python +import pandas as pd +from evidently.report import Report +from evidently.metric_preset import DataDriftPreset + +# Load reference data (e.g., training set) and current data (e.g., last week's production data) +reference_data = pd.read_csv('train.csv') +current_data = pd.read_csv('production_week1.csv') + +# Initialize the report with the DataDriftPreset +data_drift_report = Report(metrics=[DataDriftPreset()]) + +# Calculate metrics +data_drift_report.run(reference_data=reference_data, current_data=current_data) + +# Save report as HTML +data_drift_report.save_html('data_drift_report.html') + +# Get JSON output for programmatic integration (e.g., Airflow, Prefect) +drift_json = data_drift_report.json() +print(drift_json) +``` + +**Custom Tests and Thresholds**: +You can define specific tests for different features: +```python +from evidently.test_suite import TestSuite +from evidently.tests import TestNumberOfDriftedColumns, TestShareOfDriftedColumns +from evidently.tests.base_test import generate_column_tests +from evidently.tests import TestColumnDrift + +suite = TestSuite(tests=[ + TestNumberOfDriftedColumns(lt=3), # Fail if more than 2 columns drift + TestShareOfDriftedColumns(lt=0.3), # Fail if >30% of columns drift + generate_column_tests(TestColumnDrift, columns=['age', 'income']) +]) + +suite.run(reference_data=reference_data, current_data=current_data) +suite.save_html('test_suite.html') +``` + +### 4. Alerting and Retraining Strategy +Establish a workflow when drift is detected: +1. **Alerting**: Send notifications via Slack, PagerDuty, or email when drift metrics exceed thresholds. +2. **Investigation**: Data scientists analyze the report to determine if the drift is benign (e.g., seasonal change) or harmful. +3. **Retraining**: Trigger a CI/CD pipeline to retrain the model on the new data, validate the new model against a holdout set, and deploy it if it outperforms the current model. + +## Best Practices +- **Define a Baseline**: Always establish a solid baseline using the training dataset or a validated holdout set. +- **Choose the Right Window Size**: The timeframe for the "current" data depends on the business context. Daily, weekly, or monthly windows are common. +- **Monitor the Monitor**: Ensure the drift detection system itself is robust and doesn't generate excessive false positive alerts (alert fatigue). \ No newline at end of file diff --git a/TRAE-Skills/ai_engineering/Distributed_Training_Horovod.md b/TRAE-Skills/ai_engineering/Distributed_Training_Horovod.md new file mode 100644 index 0000000000000000000000000000000000000000..41ad4d8ba1018c3187a92db5f2ab8d43ab949568 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Distributed_Training_Horovod.md @@ -0,0 +1,82 @@ +# Skill: Distributed Training (Horovod) + +## Purpose +To train large machine learning models faster by distributing the computational workload across multiple GPUs or nodes, using data parallelism or model parallelism techniques. Horovod (by Uber) provides a simple and efficient framework for distributed deep learning. + +## When to Use +- When training large models (e.g., Transformers, high-res CNNs) that exceed a single GPU's memory or take too long to converge +- When scaling out training to a cluster of machines +- When utilizing multi-GPU instances in the cloud (AWS p4d, GCP A100) +- When migrating single-GPU PyTorch or TensorFlow code to multi-GPU without extensive rewrites + +## Procedure + +### 1. Installation +Install Horovod with the necessary framework support (PyTorch, TensorFlow, etc.). +```bash +# Requires MPI (Message Passing Interface) installed on the system +HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch] +``` + +### 2. PyTorch Integration Example +Convert a standard PyTorch training script to a distributed one. + +**Initialize Horovod**: +```python +import torch +import horovod.torch as hvd + +# 1. Initialize Horovod +hvd.init() + +# 2. Pin GPU to local rank (ensure each process uses a different GPU) +if torch.cuda.is_available(): + torch.cuda.set_device(hvd.local_rank()) +``` + +**Data Loading**: +Partition the dataset among workers using a `DistributedSampler`. +```python +from torch.utils.data.distributed import DistributedSampler + +dataset = MyDataset() +# 3. Partition data +sampler = DistributedSampler(dataset, num_replicas=hvd.size(), rank=hvd.rank()) +train_loader = torch.utils.data.DataLoader(dataset, batch_size=32, sampler=sampler) +``` + +**Optimizer and Broadcasting**: +Scale the learning rate by the number of workers and wrap the optimizer. +```python +model = MyModel().cuda() +# 4. Scale learning rate +optimizer = torch.optim.SGD(model.parameters(), lr=0.01 * hvd.size()) + +# 5. Add Horovod Distributed Optimizer +optimizer = hvd.DistributedOptimizer( + optimizer, named_parameters=model.named_parameters(), + op=hvd.Adsum # Default is hvd.Average +) + +# 6. Broadcast initial parameters and optimizer state from rank 0 to all other processes +hvd.broadcast_parameters(model.state_dict(), root_rank=0) +hvd.broadcast_optimizer_state(optimizer, root_rank=0) +``` + +### 3. Execution +Run the script using `horovodrun` or `mpirun`. + +**Local Multi-GPU (e.g., 4 GPUs on one machine)**: +```bash +horovodrun -np 4 -H localhost:4 python train.py +``` + +**Multi-Node (e.g., 2 machines, 4 GPUs each)**: +```bash +horovodrun -np 8 -H server1:4,server2:4 python train.py +``` + +## Best Practices +- **Learning Rate Scaling**: Always scale the learning rate linearly with the number of workers (`lr * hvd.size()`), and consider a warmup period for the first few epochs. +- **Checkpointing**: Only save checkpoints on `hvd.rank() == 0` to prevent file corruption from concurrent writes. +- **Batch Size**: The effective batch size becomes `batch_size_per_worker * hvd.size()`. Adjust accordingly to maintain convergence. \ No newline at end of file diff --git a/TRAE-Skills/ai_engineering/Embedding_Techniques.md b/TRAE-Skills/ai_engineering/Embedding_Techniques.md new file mode 100644 index 0000000000000000000000000000000000000000..35db601fadaa49c016becfd434f0b00d675c4cbf --- /dev/null +++ b/TRAE-Skills/ai_engineering/Embedding_Techniques.md @@ -0,0 +1,108 @@ +# Skill: Embedding Techniques + +## Purpose +To transform text, images, or other data into dense vector representations that capture semantic meaning for tasks like semantic search, clustering, and recommendation systems. + +## When to Use +- When building semantic search systems +- When implementing RAG (Retrieval-Augmented Generation) +- When creating recommendation engines +- When performing document similarity analysis +- When clustering content based on meaning + +## Procedure + +### 1. Choose Your Embedding Model +Select based on your use case and resource constraints. + +```python +from sentence_transformers import SentenceTransformer + +# For general purpose semantic search +model = SentenceTransformer('all-MiniLM-L6-v2') + +# For multilingual content +multilingual_model = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2') + +# For code-specific embeddings +code_model = SentenceTransformer('microsoft/codebert-base') +``` + +### 2. Generate Embeddings +Transform your documents into vectors. + +```python +documents = [ + "Machine learning is a subset of artificial intelligence", + "Deep learning uses neural networks with multiple layers", + "Natural language processing deals with text understanding" +] + +embeddings = model.encode(documents) + +print(f"Shape: {embeddings.shape}") # (3, 384) for MiniLM +print(f"Dimension: {len(embeddings[0])}") # 384 dimensions +``` + +### 3. Store in Vector Database +Persist embeddings for efficient similarity search. + +```python +import faiss +import numpy as np + +# Create FAISS index +dimension = embeddings.shape[1] +index = faiss.IndexFlatL2(dimension) + +# Add embeddings +index.add(embeddings.astype('float32')) + +# Save index +faiss.write_index(index, 'document_embeddings.index') +``` + +### 4. Semantic Search +Find similar documents using vector similarity. + +```python +query = "AI and neural networks" +query_embedding = model.encode([query]) + +# Search for top-k similar documents +k = 3 +distances, indices = index.search(query_embedding.astype('float32'), k) + +results = [(documents[i], distances[0][j]) for j, i in enumerate(indices[0])] +for doc, score in results: + print(f"Similarity: {score:.4f} | {doc}") +``` + +### 5. Batch Processing for Large Datasets +Process large document collections efficiently. + +```python +from tqdm import tqdm + +def batch_encode(documents, batch_size=32): + embeddings = [] + for i in tqdm(range(0, len(documents), batch_size)): + batch = documents[i:i + batch_size] + batch_embeddings = model.encode(batch) + embeddings.extend(batch_embeddings) + return np.array(embeddings) + +# Usage +large_corpus = load_large_dataset() # Your data loading function +embeddings = batch_encode(large_corpus, batch_size=64) +``` + +## Constraints +- **Dimensionality**: Higher dimensions = better quality but more storage/computation +- **Batch Size**: Adjust based on available GPU memory +- **Model Selection**: Consider trade-offs between quality, speed, and model size +- **Multilingual**: Use specialized models for non-English content +- **Domain-Specific**: Fine-tune or use domain-specific models for technical content + +## Expected Output +High-quality vector representations that capture semantic meaning, enabling powerful similarity search and retrieval operations. diff --git a/TRAE-Skills/ai_engineering/Federated_Learning.md b/TRAE-Skills/ai_engineering/Federated_Learning.md new file mode 100644 index 0000000000000000000000000000000000000000..535749a7dc8d72c1449adf83a26abf057d34e52f --- /dev/null +++ b/TRAE-Skills/ai_engineering/Federated_Learning.md @@ -0,0 +1,387 @@ +# Skill: Federated Learning + +## Purpose +To train machine learning models across decentralized edge devices while keeping data local, preserving privacy and reducing data transfer requirements. + +## When to Use +- When training on sensitive data that cannot leave devices +- When building healthcare or financial applications with privacy requirements +- When reducing data transfer costs in distributed systems +- When compliance with data protection regulations (GDPR, HIPAA) + +## Procedure + +### 1. Federated Learning Architecture +Set up the federated learning framework. + +```python +import torch +import torch.nn as nn +import torch.optim as optim +from copy import deepcopy +import numpy as np + +class FederatedLearningServer: + def __init__(self, global_model, learning_rate=0.01): + self.global_model = global_model + self.learning_rate = learning_rate + self.client_weights = [] + + def aggregate_models(self, client_models, client_weights=None): + """Aggregate client models using FedAvg algorithm.""" + if client_weights is None: + # Equal weights for all clients + client_weights = [1.0 / len(client_models)] * len(client_models) + + # Get global model state dict + global_state = self.global_model.state_dict() + + # Initialize aggregated state + aggregated_state = deepcopy(global_state) + for key in aggregated_state.keys(): + aggregated_state[key] = torch.zeros_like(aggregated_state[key]) + + # Aggregate weights from all clients + for client_model, weight in zip(client_models, client_weights): + client_state = client_model.state_dict() + for key in aggregated_state.keys(): + aggregated_state[key] += weight * client_state[key] + + # Update global model + self.global_model.load_state_dict(aggregated_state) + + return self.global_model + + def distribute_model(self): + """Send global model to clients.""" + return deepcopy(self.global_model) + +class FederatedLearningClient: + def __init__(self, local_model, local_data, optimizer_type='adam', learning_rate=0.01): + self.local_model = local_model + self.local_data = local_data + self.optimizer = optim.Adam(self.local_model.parameters(), lr=learning_rate) + self.criterion = nn.CrossEntropyLoss() + + def train_local(self, epochs=5, batch_size=32): + """Train model on local data.""" + self.local_model.train() + + # Assume local_data is a DataLoader + for epoch in range(epochs): + for batch_x, batch_y in self.local_data: + self.optimizer.zero_grad() + + outputs = self.local_model(batch_x) + loss = self.criterion(outputs, batch_y) + + loss.backward() + self.optimizer.step() + + return self.local_model + + def update_model(self, global_model): + """Update local model with global model.""" + self.local_model.load_state_dict(global_model.state_dict()) + return self.local_model +``` + +### 2. Differential Privacy Integration +Add privacy guarantees to federated learning. + +```python +import torch.nn.functional as F + +class DifferentialPrivacyClient: + def __init__(self, model, local_data, clip_norm=1.0, noise_multiplier=0.1): + self.model = model + self.local_data = local_data + self.clip_norm = clip_norm + self.noise_multiplier = noise_multiplier + self.optimizer = optim.Adam(self.model.parameters(), lr=0.01) + self.criterion = nn.CrossEntropyLoss() + + def clip_gradients(self): + """Clip gradients to bound sensitivity.""" + torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip_norm) + + def add_noise(self, parameters): + """Add Gaussian noise to gradients.""" + with torch.no_grad(): + for param in parameters: + noise = torch.randn_like(param) * self.noise_multiplier * self.clip_norm + param.grad += noise + + def train_with_dp(self, epochs=5): + """Train with differential privacy.""" + self.model.train() + + for epoch in range(epochs): + for batch_x, batch_y in self.local_data: + self.optimizer.zero_grad() + + outputs = self.model(batch_x) + loss = self.criterion(outputs, batch_y) + + loss.backward() + + # Apply differential privacy + self.clip_gradients() + self.add_noise(self.model.parameters()) + + self.optimizer.step() + + return self.model + + def compute_privacy_spent(self, epochs, noise_multiplier, sample_rate): + """Compute privacy budget spent.""" + # Simplified privacy accounting + # In practice, use moments accountant or RDP accountant + epsilon = epochs * sample_rate / noise_multiplier + return epsilon +``` + +### 3. Secure Aggregation +Implement secure aggregation protocols. + +```python +import hashlib +import random + +class SecureAggregationServer: + def __init__(self, global_model): + self.global_model = global_model + self.client_seeds = {} + + def distribute_seeds(self, client_ids): + """Distribute random seeds to clients.""" + seeds = {} + for client_id in client_ids: + seeds[client_id] = random.randint(0, 1000000) + self.client_seeds[client_id] = seeds[client_id] + return seeds + + def secure_aggregate(self, client_updates): + """Aggregate updates with one-time masking.""" + # In real implementation, use cryptographic protocols + # This is a simplified version + + aggregated_update = {} + + # Remove masks (in real FL, clients mask each other's updates) + for client_id, update in client_updates.items(): + for key, value in update.items(): + if key not in aggregated_update: + aggregated_update[key] = value + else: + aggregated_update[key] += value + + # Average the updates + num_clients = len(client_updates) + for key in aggregated_update: + aggregated_update[key] /= num_clients + + # Update global model + global_state = self.global_model.state_dict() + for key, value in aggregated_update.items(): + if key in global_state: + global_state[key] += value + + self.global_model.load_state_dict(global_state) + return self.global_model + +class SecureAggregationClient: + def __init__(self, model, data): + self.model = model + self.data = data + + def compute_model_update(self, global_model): + """Compute model update (difference from global model).""" + update = {} + global_state = global_model.state_dict() + local_state = self.model.state_dict() + + for key in global_state: + update[key] = local_state[key] - global_state[key] + + return update + + def apply_mask(self, update, seed): + """Apply random mask to update.""" + random.seed(seed) + masked_update = {} + + for key, value in update.items(): + # Generate random mask + mask = torch.randn_like(value) * 0.01 + masked_update[key] = value + mask + + return masked_update +``` + +### 4. Federated Learning Simulation +Simulate federated learning across multiple clients. + +```python +from torch.utils.data import DataLoader, TensorDataset +import torchvision +import torchvision.transforms as transforms + +def create_client_datasets(num_clients=10, samples_per_client=1000): + """Create local datasets for each client.""" + # Load MNIST dataset + transform = transforms.Compose([transforms.ToTensor()]) + mnist = torchvision.datasets.MNIST(root='./data', train=True, + download=True, transform=transform) + + # Split data among clients (non-IID) + client_datasets = [] + data_per_client = len(mnist) // num_clients + + for i in range(num_clients): + start_idx = i * data_per_client + end_idx = (i + 1) * data_per_client + + # Create non-IID distribution by sorting labels + data_subset = torch.utils.data.Subset(mnist, range(start_idx, end_idx)) + client_datasets.append(DataLoader(data_subset, batch_size=32, shuffle=True)) + + return client_datasets + +def run_federated_learning(num_rounds=10, num_clients=10, local_epochs=5): + """Run federated learning simulation.""" + # Create global model + global_model = nn.Sequential( + nn.Flatten(), + nn.Linear(784, 128), + nn.ReLU(), + nn.Linear(128, 10) + ) + + # Create server + server = FederatedLearningServer(global_model) + + # Create clients + client_datasets = create_client_datasets(num_clients) + clients = [] + + for data in client_datasets: + client_model = deepcopy(global_model) + client = FederatedLearningClient(client_model, data) + clients.append(client) + + # Training rounds + for round_num in range(num_rounds): + print(f"Round {round_num + 1}/{num_rounds}") + + # Distribute global model + global_model_state = server.distribute_model() + + # Select subset of clients (client sampling) + selected_clients = random.sample(clients, min(5, len(clients))) + + # Local training + client_models = [] + for client in selected_clients: + client.update_model(global_model_state) + updated_model = client.train_local(epochs=local_epochs) + client_models.append(updated_model) + + # Aggregate models + server.aggregate_models(client_models) + + # Evaluate global model (simplified) + print(f" Completed training with {len(client_models)} clients") + + return global_model + +# Usage +# global_model = run_federated_learning(num_rounds=10, num_clients=10, local_epochs=5) +# print("Federated learning completed!") +``` + +### 5. Monitoring and Evaluation +Monitor federated learning progress. + +```python +class FederatedLearningMonitor: + def __init__(self): + self.metrics = { + 'round': [], + 'client_accuracies': [], + 'global_accuracy': [], + 'communication_cost': [], + 'privacy_budget': [] + } + + def log_round(self, round_num, client_metrics, global_accuracy, comm_cost, privacy_epsilon): + """Log metrics for a round.""" + self.metrics['round'].append(round_num) + self.metrics['client_accuracies'].append(client_metrics) + self.metrics['global_accuracy'].append(global_accuracy) + self.metrics['communication_cost'].append(comm_cost) + self.metrics['privacy_budget'].append(privacy_epsilon) + + def evaluate_global_model(self, model, test_loader): + """Evaluate global model on test data.""" + model.eval() + correct = 0 + total = 0 + + with torch.no_grad(): + for batch_x, batch_y in test_loader: + outputs = model(batch_x) + _, predicted = torch.max(outputs.data, 1) + total += batch_y.size(0) + correct += (predicted == batch_y).sum().item() + + accuracy = 100 * correct / total + return accuracy + + def generate_report(self): + """Generate training report.""" + import matplotlib.pyplot as plt + + # Plot global accuracy over rounds + plt.figure(figsize=(12, 4)) + + plt.subplot(1, 3, 1) + plt.plot(self.metrics['round'], self.metrics['global_accuracy']) + plt.xlabel('Round') + plt.ylabel('Global Accuracy (%)') + plt.title('Global Model Performance') + + plt.subplot(1, 3, 2) + plt.plot(self.metrics['round'], self.metrics['communication_cost']) + plt.xlabel('Round') + plt.ylabel('Communication Cost (MB)') + plt.title('Communication Overhead') + + plt.subplot(1, 3, 3) + plt.plot(self.metrics['round'], self.metrics['privacy_budget']) + plt.xlabel('Round') + plt.ylabel('Privacy Budget (ε)') + plt.title('Privacy Budget Consumption') + + plt.tight_layout() + plt.savefig('federated_learning_metrics.png') + plt.close() + + return "Federated learning metrics saved to federated_learning_metrics.png" + +# Usage +# monitor = FederatedLearningMonitor() +# monitor.log_round(1, [85.2, 87.1, 86.5], 86.0, 5.2, 0.1) +# monitor.generate_report() +``` + +## Constraints +- **Communication Overhead**: Frequent model updates can be bandwidth-intensive +- **Heterogeneity**: Non-IID data across clients can hurt convergence +- **Privacy-Utility Tradeoff**: Stronger privacy protection may reduce model accuracy +- **System Complexity**: Federated learning systems are complex to implement and maintain +- **Client Availability**: Clients may be unavailable or have varying capabilities +- **Scalability**: Large numbers of clients present coordination challenges + +## Expected Output +A privacy-preserving machine learning system that trains models across decentralized devices while maintaining data locality and providing strong privacy guarantees. diff --git a/TRAE-Skills/ai_engineering/Fine_Tuning_Custom_Models.md b/TRAE-Skills/ai_engineering/Fine_Tuning_Custom_Models.md new file mode 100644 index 0000000000000000000000000000000000000000..62e139d1347cad5d1526db0da335fbaecb3d0018 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Fine_Tuning_Custom_Models.md @@ -0,0 +1,237 @@ +# Skill: Fine-Tuning Custom Models + +## Purpose +To adapt pre-trained LLMs to specific domains, tasks, or styles by training on custom datasets for improved performance on specialized use cases. + +## When to Use +- When you need domain-specific knowledge (legal, medical, technical) +- When you want a specific writing style or tone +- When you have a large dataset of example inputs/outputs +- When you need consistent formatting or structure in outputs + +## Procedure + +### 1. Prepare Your Dataset +Create properly formatted training data. + +```python +import json + +# Training data format for OpenAI fine-tuning +training_data = [ + { + "messages": [ + {"role": "system", "content": "You are a legal assistant specializing in contract review."}, + {"role": "user", "content": "Review this clause: 'The company may terminate at any time.'"}, + {"role": "assistant", "content": "This clause is one-sided. Consider adding: 'either party may terminate with 30 days notice.'"} + ] + }, + { + "messages": [ + {"role": "system", "content": "You are a legal assistant specializing in contract review."}, + {"role": "user", "content": "Is this confidentiality clause standard?"}, + {"role": "assistant", "content": "The clause appears standard. It includes mutual confidentiality obligations and defines exceptions appropriately."} + ] + } +] + +# Save to JSONL file +with open('training_data.jsonl', 'w') as f: + for entry in training_data: + f.write(json.dumps(entry) + '\n') +``` + +### 2. Validate Your Data +Ensure data quality before training. + +```python +import json +from collections import defaultdict + +def validate_finetuning_data(file_path): + """Validate fine-tuning dataset.""" + errors = [] + stats = defaultdict(int) + + with open(file_path, 'r') as f: + for line_num, line in enumerate(f, 1): + try: + entry = json.loads(line) + stats['total_entries'] += 1 + + # Check required fields + if 'messages' not in entry: + errors.append(f"Line {line_num}: Missing 'messages' field") + continue + + messages = entry['messages'] + stats['total_messages'] += len(messages) + + # Validate message structure + for msg in messages: + if 'role' not in msg or 'content' not in msg: + errors.append(f"Line {line_num}: Invalid message structure") + + if msg['role'] not in ['system', 'user', 'assistant']: + errors.append(f"Line {line_num}: Invalid role: {msg['role']}") + + stats[f"role_{msg['role']}"] += 1 + + except json.JSONDecodeError: + errors.append(f"Line {line_num}: Invalid JSON") + + return { + 'errors': errors, + 'stats': dict(stats) + } + +# Usage +validation_result = validate_finetuning_data('training_data.jsonl') +print(f"Validation complete: {validation_result['stats']['total_entries']} entries") +if validation_result['errors']: + print(f"Errors found: {len(validation_result['errors'])}") + for error in validation_result['errors'][:10]: # Show first 10 errors + print(f" - {error}") +``` + +### 3. Upload and Prepare Training +Upload data to OpenAI and start fine-tuning. + +```python +from openai import OpenAI + +client = OpenAI() + +# Upload training file +with open('training_data.jsonl', 'rb') as f: + upload_response = client.files.create( + file=f, + purpose='fine-tune' + ) + +training_file_id = upload_response.id +print(f"File uploaded: {training_file_id}") + +# Create fine-tuning job +fine_tune_job = client.fine_tuning.jobs.create( + training_file=training_file_id, + model="gpt-3.5-turbo", + hyperparameters={ + "n_epochs": 3, + "batch_size": 4, + "learning_rate_multiplier": 0.1 + } +) + +job_id = fine_tune_job.id +print(f"Fine-tuning job started: {job_id}") +``` + +### 4. Monitor Training Progress +Track the fine-tuning process. + +```python +def check_finetuning_status(job_id): + """Check the status of a fine-tuning job.""" + job = client.fine_tuning.jobs.retrieve(job_id) + + status = { + 'status': job.status, + 'created_at': job.created_at, + 'finished_at': job.finished_at, + 'model': job.fine_tuned_model, + 'error': job.error + } + + if job.result_files: + # Retrieve training metrics + result_file = client.files.retrieve(job.result_files[0]) + print(f"Results file: {result_file.id}") + + return status + +# Usage +import time + +while True: + status = check_finetuning_status(job_id) + print(f"Status: {status['status']}") + + if status['status'] in ['succeeded', 'failed', 'cancelled']: + break + + time.sleep(60) # Check every minute +``` + +### 5. Use the Fine-Tuned Model +Deploy and use your custom model. + +```python +def use_finetuned_model(model_id, prompt): + """Use the fine-tuned model for inference.""" + response = client.chat.completions.create( + model=model_id, + messages=[{"role": "user", "content": prompt}] + ) + return response.choices[0].message.content + +# After training completes +fine_tuned_model = status['model'] + +# Test the model +test_prompt = "Review this contract clause: 'Employee shall not compete for 2 years after termination.'" +result = use_finetuned_model(fine_tuned_model, test_prompt) +print(result) +``` + +### 6. Evaluate Model Performance +Assess the quality of your fine-tuned model. + +```python +def evaluate_model(model_id, test_data): + """Evaluate fine-tuned model on test set.""" + results = [] + + for test_case in test_data: + # Get model response + response = use_finetuned_model(model_id, test_case['input']) + + # Compare with expected output + results.append({ + 'input': test_case['input'], + 'expected': test_case['expected'], + 'actual': response, + 'match': test_case['expected'].lower() in response.lower() + }) + + # Calculate metrics + accuracy = sum(r['match'] for r in results) / len(results) + + return { + 'accuracy': accuracy, + 'results': results + } + +# Test dataset +test_data = [ + { + 'input': 'Is this termination clause fair?', + 'expected': 'fair' # or 'unfair' + } + # ... more test cases +] + +evaluation = evaluate_model(fine_tuned_model, test_data) +print(f"Accuracy: {evaluation['accuracy']:.2%}") +``` + +## Constraints +- **Minimum Data**: Need at least 10-100 examples for meaningful fine-tuning +- **Data Quality**: Garbage in, garbage out - ensure high-quality training data +- **Cost**: Fine-tuning can be expensive, especially with large datasets +- **Overfitting**: Monitor for overfitting to training data +- **Hallucination**: Fine-tuned models may still hallucinate facts +- **Maintenance**: Models may need periodic retraining as data evolves + +## Expected Output +A specialized model that performs better on your specific domain tasks compared to base models, with consistent formatting and domain-specific knowledge. diff --git a/TRAE-Skills/ai_engineering/Fine_tuning_Basics.md b/TRAE-Skills/ai_engineering/Fine_tuning_Basics.md new file mode 100644 index 0000000000000000000000000000000000000000..1384524eacde078c622b4296d19f7b0d4ee389f6 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Fine_tuning_Basics.md @@ -0,0 +1,85 @@ +# Skill: Fine-tuning Basics + +## Purpose +To adapt a pre-trained LLM to a specific task, tone, or domain by training it on a specialized dataset, ensuring rigid adherence to format or style. + +## When to Use +- When "Prompt Engineering" fails to produce the desired format consistently. +- When you need to mimic a very specific brand voice or writing style. +- To reduce latency and costs by using a smaller model (e.g., GPT-4o-mini) that performs like a larger one on a specific task. + +## Procedure + +### 1. Data Preparation (JSONL) +Create a dataset in the specific format required by the provider (OpenAI example). + +```json +// training_data.jsonl +{"messages": [{"role": "system", "content": "You are a code reviewer."}, {"role": "user", "content": "Review this: const x = 1;"}, {"role": "assistant", "content": "LGTM. Consider using 'let' if reassigning."}]} +{"messages": [{"role": "system", "content": "You are a code reviewer."}, {"role": "user", "content": "Review this: alert('hi');"}, {"role": "assistant", "content": "Avoid 'alert' in production code."}]} +``` + +### 2. Validation Script (Python) +Always validate your JSONL before uploading to avoid costly training failures. + +```python +import json + +def validate_data(file_path): + with open(file_path, 'r') as f: + for line in f: + try: + data = json.loads(line) + if "messages" not in data: + print("Missing 'messages' key") + except Exception as e: + print(f"Error parsing line: {e}") + +validate_data("training_data.jsonl") +``` + +### 3. Starting the Fine-tuning Job (OpenAI CLI) +Upload the file and start the training process. + +```bash +# Install CLI +pip install openai + +# Set Key +export OPENAI_API_KEY="your-key" + +# Upload file +openai files create -f training_data.jsonl -p fine-tune + +# Start training +openai fine_tuning.jobs.create -t "file-id-from-upload" -m "gpt-4o-mini-2024-07-18" +``` + +### 4. Monitoring & Evaluation +Check the status and loss metrics. + +```bash +# List jobs +openai fine_tuning.jobs.list + +# Retrieve status +openai fine_tuning.jobs.retrieve -i "ft-job-id" +``` + +### 5. Using the Fine-tuned Model +Once completed, use the new model ID in your application. + +```typescript +const completion = await openai.chat.completions.create({ + model: "ft:gpt-4o-mini:your-org:custom-name:id", + messages: [{ role: "user", content: "Review this: console.log(1);" }], +}); +``` + +## Constraints +- **Overfitting**: Don't train for too many epochs; the model will lose its general intelligence. +- **Facts vs Style**: **Never** fine-tune to teach new facts. Use RAG for facts. Use fine-tuning for **how** the model speaks. +- **Minimum Data**: You need at least 50-100 high-quality examples to see any meaningful improvement. + +## Expected Output +A specialized model ID that delivers high-performance results on a specific, narrow task. diff --git a/TRAE-Skills/ai_engineering/Function_Calling.md b/TRAE-Skills/ai_engineering/Function_Calling.md new file mode 100644 index 0000000000000000000000000000000000000000..939a66bcbde66e66ba6a06a48945efca3d2d6aaf --- /dev/null +++ b/TRAE-Skills/ai_engineering/Function_Calling.md @@ -0,0 +1,280 @@ +# Skill: Function Calling + +## Purpose +To enable LLMs to interact with external tools, APIs, and databases by defining structured function specifications that models can intelligently invoke. + +## When to Use +- When building AI assistants that need to perform actions +- When integrating LLMs with existing APIs +- When requiring structured data extraction +- When implementing multi-step workflows with external systems + +## Procedure + +### 1. Define Function Schemas +Create structured function definitions with clear descriptions. + +```python +from openai import OpenAI + +client = OpenAI() + +# Define available functions +functions = [ + { + "name": "get_weather", + "description": "Get the current weather for a specific location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit" + } + }, + "required": ["location"] + } + }, + { + "name": "search_database", + "description": "Search a database for specific records", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The search query" + }, + "limit": { + "type": "integer", + "description": "Maximum number of results to return" + } + }, + "required": ["query"] + } + } +] +``` + +### 2. Implement the Functions +Create actual implementations of your defined functions. + +```python +import requests + +def get_weather(location, unit="celsius"): + """Get weather data for a location.""" + # Example using a weather API + api_url = f"https://api.weatherapi.com/v1/current.json" + params = { + "key": "your-api-key", + "q": location, + "aqi": "no" + } + + try: + response = requests.get(api_url, params=params) + data = response.json() + + temp = data['current']['temp_c'] if unit == "celsius" else data['current']['temp_f'] + condition = data['current']['condition']['text'] + + return { + "location": location, + "temperature": temp, + "unit": unit, + "condition": condition + } + except Exception as e: + return {"error": str(e)} + +def search_database(query, limit=10): + """Search database for records.""" + # Example database query + # In production, use your actual database connection + results = [ + {"id": 1, "title": f"Result for {query}", "content": "..."}, + {"id": 2, "title": f"Another result for {query}", "content": "..."} + ] + return results[:limit] + +# Map function names to implementations +function_map = { + "get_weather": get_weather, + "search_database": search_database +} +``` + +### 3. Handle Function Calls +Process LLM responses that request function calls. + +```python +def execute_function_call(function_call): + """Execute a function call from the LLM.""" + function_name = function_call.name + function_args = json.loads(function_call.arguments) + + print(f"Calling function: {function_name}") + print(f"Arguments: {function_args}") + + if function_name in function_map: + function_to_call = function_map[function_name] + return function_to_call(**function_args) + else: + return f"Error: Function {function_name} not found" +``` + +### 4. Complete Conversation Flow +Implement the full interaction loop. + +```python +def chat_with_functions(user_message, messages_history=None): + """Handle conversation with function calling.""" + if messages_history is None: + messages_history = [] + + # Add user message to history + messages_history.append({"role": "user", "content": user_message}) + + # Make API call with functions + response = client.chat.completions.create( + model="gpt-4", + messages=messages_history, + functions=functions, + function_call="auto" # Let model decide whether to call functions + ) + + response_message = response.choices[0].message + + # Check if model wants to call a function + if response_message.function_call: + print(f"Model wants to call: {response_message.function_call.name}") + + # Execute function call + function_response = execute_function_call(response_message.function_call) + + # Add function response to conversation + messages_history.append(response_message) # Assistant message with function call + messages_history.append({ + "role": "function", + "name": response_message.function_call.name, + "content": json.dumps(function_response) + }) + + # Get final response from model + second_response = client.chat.completions.create( + model="gpt-4", + messages=messages_history + ) + + return second_response.choices[0].message.content + else: + return response_message.content + +# Usage +messages = [] +while True: + user_input = input("You: ") + if user_input.lower() in ['quit', 'exit']: + break + + response = chat_with_functions(user_input, messages) + print(f"Assistant: {response}") + messages.append({"role": "assistant", "content": response}) +``` + +### 5. Advanced: Parallel Function Calls +Handle multiple function calls in a single request. + +```python +def handle_parallel_function_calls(response_message, messages_history): + """Handle multiple function calls from the model.""" + # Some models can request multiple function calls at once + if hasattr(response_message, 'function_calls'): + function_calls = response_message.function_calls + else: + # Single function call (convert to list for uniform handling) + if response_message.function_call: + function_calls = [response_message.function_call] + else: + return messages_history + + # Add assistant message with function calls + messages_history.append(response_message) + + # Execute all function calls + for function_call in function_calls: + function_response = execute_function_call(function_call) + + # Add each function response + messages_history.append({ + "role": "function", + "name": function_call.name, + "content": json.dumps(function_response) + }) + + return messages_history +``` + +### 6. Error Handling and Validation +Add robust error handling for function calls. + +```python +def safe_execute_function_call(function_call): + """Execute function call with error handling.""" + function_name = function_call.name + + try: + function_args = json.loads(function_call.arguments) + except json.JSONDecodeError as e: + return { + "error": f"Invalid JSON in arguments: {str(e)}", + "arguments": function_call.arguments + } + + # Validate function exists + if function_name not in function_map: + return { + "error": f"Unknown function: {function_name}", + "available_functions": list(function_map.keys()) + } + + # Validate required parameters + function_schema = next((f for f in functions if f["name"] == function_name), None) + if function_schema: + required_params = function_schema["parameters"].get("required", []) + missing_params = [p for p in required_params if p not in function_args] + + if missing_params: + return { + "error": f"Missing required parameters: {', '.join(missing_params)}", + "provided": list(function_args.keys()) + } + + # Execute function + try: + function_to_call = function_map[function_name] + return function_to_call(**function_args) + except Exception as e: + return { + "error": f"Function execution failed: {str(e)}", + "function": function_name, + "arguments": function_args + } +``` + +## Constraints +- **Function Descriptions**: Clear, detailed descriptions are crucial for model performance +- **Parameter Validation**: Always validate function arguments before execution +- **Error Handling**: Gracefully handle function failures and API errors +- **Security**: Validate and sanitize all inputs, especially for database operations +- **Rate Limiting**: Implement rate limits for external API calls +- **Token Limits**: Function calls consume tokens - consider size of function schemas and responses + +## Expected Output +An intelligent AI assistant capable of understanding user intent and automatically executing appropriate functions to fulfill requests, with proper error handling and response formatting. diff --git a/TRAE-Skills/ai_engineering/Generative_AI_Image_Synthesis.md b/TRAE-Skills/ai_engineering/Generative_AI_Image_Synthesis.md new file mode 100644 index 0000000000000000000000000000000000000000..a7c869788c2fabb7cb7d6f55e2edca871cced8f1 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Generative_AI_Image_Synthesis.md @@ -0,0 +1,96 @@ +# Skill: Generative AI Image Synthesis + +## Purpose +To create and integrate generative image AI models (Stable Diffusion, DALL-E, Midjourney API) into applications for generating, editing, and manipulating images programmatically. + +## When to Use +- When building applications that require on-demand image generation from text prompts +- For creating custom image editing tools using AI inpainting/outpainting +- When implementing AI-powered design tools for creative industries +- For generating synthetic datasets for computer vision tasks + +## Procedure + +### 1. Stable Diffusion Integration (Hugging Face) +Use the Hugging Face Diffusers library for local Stable Diffusion generation. + +```python +from diffusers import StableDiffusionPipeline +import torch + +model_id = "runwayml/stable-diffusion-v1-5" +pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16) +pipe = pipe.to("cuda") + +prompt = "A majestic lion standing on a cliff at sunset, photorealistic" +image = pipe(prompt).images[0] +image.save("lion_sunset.png") +``` + +### 2. OpenAI DALL-E API Integration +Use OpenAI's API for cloud-based image generation. + +```javascript +import OpenAI from 'openai'; + +const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); + +async function generateImage(prompt) { + const response = await openai.images.generate({ + model: 'dall-e-3', + prompt: prompt, + n: 1, + size: '1024x1024', + quality: 'standard', + }); + return response.data[0].url; +} +``` + +### 3. Inpainting for Image Editing +Modify specific regions of existing images. + +```python +from diffusers import StableDiffusionInpaintPipeline +from PIL import Image +import torch + +pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", + torch_dtype=torch.float16 +).to("cuda") + +image = Image.open("original_image.jpg") +mask = Image.open("mask_image.png") + +prompt = "A cute dog sitting on the couch" +result = pipe(prompt=prompt, image=image, mask_image=mask).images[0] +result.save("edited_image.jpg") +``` + +### 4. Image-to-Image Translation +Transform existing images based on text prompts. + +```python +from diffusers import StableDiffusionImg2ImgPipeline +from PIL import Image +import torch + +pipe = StableDiffusionImg2ImgPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + torch_dtype=torch.float16 +).to("cuda") + +init_image = Image.open("sketch.jpg").convert("RGB") +prompt = "A realistic watercolor painting of a mountain landscape" + +result = pipe(prompt=prompt, image=init_image, strength=0.75, guidance_scale=7.5).images[0] +result.save("watercolor_mountain.jpg") +``` + +## Best Practices +- **Prompt Engineering**: Be specific about style, lighting, composition, and artists for better results +- **Memory Management**: Use float16 and model quantization to reduce GPU memory usage +- **Caching**: Cache frequently used generated images to reduce API costs +- **Content Moderation**: Implement safety filters to prevent generation of inappropriate content +- **Rate Limiting**: Respect API rate limits and implement retry logic with exponential backoff diff --git a/TRAE-Skills/ai_engineering/LLM_Caching_Strategies.md b/TRAE-Skills/ai_engineering/LLM_Caching_Strategies.md new file mode 100644 index 0000000000000000000000000000000000000000..2955b2b76b36d3ea4bc86e90a4eb5f2f2118503c --- /dev/null +++ b/TRAE-Skills/ai_engineering/LLM_Caching_Strategies.md @@ -0,0 +1,259 @@ +# Skill: LLM Caching Strategies + +## Purpose +To reduce API costs and latency by caching LLM responses and avoiding redundant calls for identical or similar prompts. + +## When to Use +- When users frequently ask similar questions +- When implementing RAG systems with recurring queries +- When building applications with repetitive patterns +- When optimizing for cost reduction + +## Procedure + +### 1. Simple Exact Match Caching +Cache responses based on exact prompt matches. + +```python +import hashlib +import json +from functools import wraps +from openai import OpenAI + +client = OpenAI() + +# Simple in-memory cache +response_cache = {} + +def cache_key(prompt, model, temperature=0): + """Generate a unique cache key.""" + content = f"{model}:{temperature}:{prompt}" + return hashlib.sha256(content.encode()).hexdigest() + +def cached_completion(prompt, model="gpt-4", temperature=0): + """Get completion with caching.""" + key = cache_key(prompt, model, temperature) + + if key in response_cache: + print("Cache hit!") + return response_cache[key] + + print("Cache miss - calling API...") + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=temperature + ) + + result = response.choices[0].message.content + response_cache[key] = result + return result +``` + +### 2. Semantic Caching with Embeddings +Cache responses based on semantic similarity. + +```python +import numpy as np +from sentence_transformers import SentenceTransformer + +class SemanticCache: + def __init__(self, similarity_threshold=0.85): + self.embedder = SentenceTransformer('all-MiniLM-L6-v2') + self.prompts = [] + self.responses = [] + self.embeddings = [] + self.threshold = similarity_threshold + + def get(self, prompt): + """Try to get a cached response based on semantic similarity.""" + if not self.prompts: + return None + + # Embed the query prompt + query_embedding = self.embedder.encode([prompt])[0] + + # Calculate similarities with cached prompts + cached_embeddings = np.array(self.embeddings) + similarities = np.dot(cached_embeddings, query_embedding) / ( + np.linalg.norm(cached_embeddings, axis=1) * np.linalg.norm(query_embedding) + ) + + # Find the most similar cached prompt + max_idx = np.argmax(similarities) + max_similarity = similarities[max_idx] + + if max_similarity >= self.threshold: + print(f"Semantic cache hit! Similarity: {max_similarity:.3f}") + return self.responses[max_idx] + + print(f"No semantic match found. Max similarity: {max_similarity:.3f}") + return None + + def set(self, prompt, response): + """Store a new prompt-response pair.""" + self.prompts.append(prompt) + self.responses.append(response) + self.embeddings.append(self.embedder.encode([prompt])[0]) + +# Usage +semantic_cache = SemanticCache(similarity_threshold=0.90) + +def get_response_with_semantic_cache(prompt): + # Check cache first + cached = semantic_cache.get(prompt) + if cached: + return cached + + # Call API and cache the result + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}] + ) + result = response.choices[0].message.content + + semantic_cache.set(prompt, result) + return result +``` + +### 3. Persistent Caching with Redis +Store cache entries in Redis for persistence across restarts. + +```python +import redis +import pickle +import hashlib + +redis_client = redis.Redis(host='localhost', port=6379, db=0) + +def redis_cache_key(prompt, model="gpt-4"): + """Generate Redis cache key.""" + content = f"llm_cache:{model}:{hashlib.sha256(prompt.encode()).hexdigest()}" + return content + +def get_with_redis_cache(prompt, model="gpt-4", expire_hours=24): + """Get response with Redis caching.""" + key = redis_cache_key(prompt, model) + + # Try to get from Redis + cached = redis_client.get(key) + if cached: + print("Redis cache hit!") + return pickle.loads(cached) + + print("Redis cache miss - calling API...") + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}] + ) + result = response.choices[0].message.content + + # Store in Redis with expiration + redis_client.setex(key, expire_hours * 3600, pickle.dumps(result)) + return result +``` + +### 4. Hierarchical Caching +Combine multiple caching strategies for optimal performance. + +```python +class HierarchicalCache: + def __init__(self): + self.memory_cache = {} # L1: In-memory cache + self.semantic_cache = SemanticCache() # L2: Semantic cache + self.redis_client = redis.Redis() # L3: Persistent cache + + def get(self, prompt, model="gpt-4"): + # Check L1: Exact match in memory + key = cache_key(prompt, model) + if key in self.memory_cache: + return self.memory_cache[key] + + # Check L2: Semantic match + semantic_result = self.semantic_cache.get(prompt) + if semantic_result: + self.memory_cache[key] = semantic_result + return semantic_result + + # Check L3: Redis persistent cache + redis_key = redis_cache_key(prompt, model) + redis_result = self.redis_client.get(redis_key) + if redis_result: + result = pickle.loads(redis_result) + self.memory_cache[key] = result + self.semantic_cache.set(prompt, result) + return result + + # Cache miss - call API + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}] + ) + result = response.choices[0].message.content + + # Store at all levels + self.memory_cache[key] = result + self.semantic_cache.set(prompt, result) + self.redis_client.setex(redis_key, 86400, pickle.dumps(result)) + + return result + +# Usage +hierarchical_cache = HierarchicalCache() +response = hierarchical_cache.get("Explain quantum computing") +``` + +### 5. Cache Statistics and Monitoring +Track cache performance to optimize strategies. + +```python +from collections import defaultdict + +class CacheWithStats: + def __init__(self): + self.cache = {} + self.stats = defaultdict(int) + + def get(self, key): + self.stats['total_requests'] += 1 + + if key in self.cache: + self.stats['cache_hits'] += 1 + self.stats['memory_cache_hits'] += 1 + return self.cache[key] + + self.stats['cache_misses'] += 1 + return None + + def set(self, key, value): + self.cache[key] = value + self.stats['items_cached'] += 1 + + def get_stats(self): + total = self.stats['total_requests'] + hits = self.stats['cache_hits'] + hit_rate = (hits / total * 100) if total > 0 else 0 + + return { + 'total_requests': total, + 'cache_hits': hits, + 'cache_misses': self.stats['cache_misses'], + 'hit_rate': f"{hit_rate:.2f}%", + 'items_cached': self.stats['items_cached'] + } + +# Usage +cache = CacheWithStats() +# ... perform operations ... +print(cache.get_stats()) +``` + +## Constraints +- **Memory Usage**: Semantic caching stores embeddings in memory +- **Staleness**: Cached responses may become outdated +- **Similarity Threshold**: Tune based on your use case (0.85-0.95) +- **Cache Size**: Implement cache eviction policies for long-running systems +- **Cost vs. Freshness**: Balance between caching and getting fresh responses + +## Expected Output +Significant reduction in API costs (50-90% in some cases) and improved latency through intelligent caching of LLM responses. diff --git a/TRAE-Skills/ai_engineering/LLM_Function_Calling_Advanced.md b/TRAE-Skills/ai_engineering/LLM_Function_Calling_Advanced.md new file mode 100644 index 0000000000000000000000000000000000000000..e3d3ce6b3919037b9e90148b4758b1cdc7b8d046 --- /dev/null +++ b/TRAE-Skills/ai_engineering/LLM_Function_Calling_Advanced.md @@ -0,0 +1,234 @@ +# Skill: Advanced LLM Function Calling + +## Purpose +To implement advanced function calling patterns with Large Language Models, enabling them to interact with external APIs, databases, and tools in a structured, safe way. + +## When to Use +- When building AI agents that need to take real-world actions +- For integrating LLMs with existing APIs and services +- When you need structured, deterministic outputs from LLMs +- For building copilots and assistant applications +- When implementing multi-step reasoning and tool use + +## Procedure + +### 1. Function Definition Schema +Define functions with clear descriptions and schemas. + +```javascript +import OpenAI from 'openai'; + +const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); + +const functions = [ + { + name: 'get_weather', + description: 'Get the current weather in a given location', + parameters: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city and state, e.g., San Francisco, CA', + }, + unit: { + type: 'string', + enum: ['celsius', 'fahrenheit'], + description: 'The temperature unit to use', + }, + }, + required: ['location'], + }, + }, + { + name: 'search_products', + description: 'Search for products in the catalog', + parameters: { + type: 'object', + properties: { + query: { + type: 'string', + description: 'The search query', + }, + category: { + type: 'string', + enum: ['electronics', 'clothing', 'books'], + description: 'Product category', + }, + max_results: { + type: 'integer', + description: 'Maximum number of results to return', + }, + }, + required: ['query'], + }, + }, +]; +``` + +### 2. Function Calling Loop +Implement a loop to handle multiple function calls. + +```javascript +async function chatWithFunctions(messages) { + let response = await openai.chat.completions.create({ + model: 'gpt-4', + messages, + functions, + function_call: 'auto', + }); + + let message = response.choices[0].message; + + while (message.function_call) { + const functionName = message.function_call.name; + const functionArgs = JSON.parse(message.function_call.arguments); + + // Execute the function + let functionResponse; + switch (functionName) { + case 'get_weather': + functionResponse = await getWeather(functionArgs); + break; + case 'search_products': + functionResponse = await searchProducts(functionArgs); + break; + default: + functionResponse = { error: `Unknown function: ${functionName}` }; + } + + // Add the function response to messages + messages.push(message); + messages.push({ + role: 'function', + name: functionName, + content: JSON.stringify(functionResponse), + }); + + // Get next response from LLM + response = await openai.chat.completions.create({ + model: 'gpt-4', + messages, + functions, + function_call: 'auto', + }); + + message = response.choices[0].message; + } + + return message.content; +} + +// Usage +const messages = [ + { role: 'user', content: 'What is the weather in New York and show me 3 electronics products?' } +]; + +const result = await chatWithFunctions(messages); +console.log(result); +``` + +### 3. Parallel Function Calling +Execute multiple functions in parallel. + +```javascript +async function chatWithParallelFunctions(messages) { + let response = await openai.chat.completions.create({ + model: 'gpt-4', + messages, + functions, + function_call: 'auto', + }); + + let message = response.choices[0].message; + + while (message.function_call || message.tool_calls) { + const calls = message.tool_calls || [message.function_call]; + + // Execute all functions in parallel + const functionResponses = await Promise.all( + calls.map(async (call) => { + const funcCall = call.type === 'function' ? call.function : call; + const name = funcCall.name; + const args = JSON.parse(funcCall.arguments); + + let result; + switch (name) { + case 'get_weather': + result = await getWeather(args); + break; + case 'search_products': + result = await searchProducts(args); + break; + default: + result = { error: `Unknown function: ${name}` }; + } + + return { + id: call.id, + role: 'tool', + name: name, + content: JSON.stringify(result), + }; + }) + ); + + messages.push(message); + messages.push(...functionResponses); + + response = await openai.chat.completions.create({ + model: 'gpt-4', + messages, + functions, + function_call: 'auto', + }); + + message = response.choices[0].message; + } + + return message.content; +} +``` + +### 4. Safety & Validation +Validate function inputs before execution. + +```javascript +import { z } from 'zod'; + +const GetWeatherSchema = z.object({ + location: z.string().min(2), + unit: z.enum(['celsius', 'fahrenheit']).optional().default('celsius'), +}); + +async function getWeatherSafe(args) { + try { + const validatedArgs = GetWeatherSchema.parse(args); + return await getWeather(validatedArgs); + } catch (error) { + return { error: 'Invalid arguments', details: error.message }; + } +} + +// Add authentication and authorization +async function executeFunction(name, args, userId) { + // Check if user has permission to use this function + if (!hasPermission(userId, name)) { + return { error: 'Permission denied' }; + } + + // Validate arguments + // Execute function + // Log the function call for auditing + logFunctionCall(userId, name, args); +} +``` + +## Best Practices +- **Clear Descriptions**: Write clear, detailed descriptions for functions and parameters +- **Input Validation**: Always validate function inputs before execution +- **Error Handling**: Gracefully handle errors and communicate them back to the LLM +- **Safety**: Implement authentication, authorization, and rate limiting +- **Idempotency**: Make functions idempotent when possible +- **Logging**: Log all function calls for debugging and auditing +- **Context Limit**: Be mindful of context limits and manage conversation history diff --git a/TRAE-Skills/ai_engineering/LLM_Operations.md b/TRAE-Skills/ai_engineering/LLM_Operations.md new file mode 100644 index 0000000000000000000000000000000000000000..5434887052b63881da6797b21720f92d6036d4b7 --- /dev/null +++ b/TRAE-Skills/ai_engineering/LLM_Operations.md @@ -0,0 +1,507 @@ +# Skill: LLM Operations (LLMOps) + +## Purpose +To operationalize large language models in production environments with proper deployment, scaling, monitoring, and maintenance. + +## When to Use +- When deploying LLMs to production +- When managing multiple LLM deployments +- When optimizing LLM performance and costs +- When implementing LLM version control and rollback + +## Procedure + +### 1. Model Deployment Strategy +Implement robust deployment strategies for LLMs. + +```python +from abc import ABC, abstractmethod +from openai import OpenAI +import time +from functools import wraps +import logging + +class LLMProvider(ABC): + """Abstract base class for LLM providers.""" + + @abstractmethod + def generate(self, prompt, **kwargs): + pass + + @abstractmethod + def health_check(self): + pass + +class OpenAIProvider(LLMProvider): + def __init__(self, api_key, model="gpt-4"): + self.client = OpenAI(api_key=api_key) + self.model = model + self.logger = logging.getLogger(__name__) + + def generate(self, prompt, temperature=0.7, max_tokens=1000): + """Generate text using OpenAI API.""" + try: + start_time = time.time() + + response = self.client.chat.completions.create( + model=self.model, + messages=[{"role": "user", "content": prompt}], + temperature=temperature, + max_tokens=max_tokens + ) + + latency = time.time() - start_time + + result = { + 'text': response.choices[0].message.content, + 'tokens_used': response.usage.total_tokens, + 'latency': latency, + 'model': self.model + } + + self.logger.info(f"Generated {result['tokens_used']} tokens in {latency:.2f}s") + return result + + except Exception as e: + self.logger.error(f"Generation failed: {str(e)}") + raise + + def health_check(self): + """Check if the API is accessible.""" + try: + response = self.client.chat.completions.create( + model=self.model, + messages=[{"role": "user", "content": "ping"}], + max_tokens=5 + ) + return {'status': 'healthy', 'model': self.model} + except Exception as e: + return {'status': 'unhealthy', 'error': str(e)} + +class LocalLLMProvider(LLMProvider): + """Provider for locally hosted LLMs (e.g., using Ollama or vLLM).""" + + def __init__(self, endpoint, model_name): + self.endpoint = endpoint + self.model_name = model_name + self.logger = logging.getLogger(__name__) + + def generate(self, prompt, temperature=0.7, max_tokens=1000): + """Generate text using local LLM.""" + import requests + + try: + start_time = time.time() + + response = requests.post( + f"{self.endpoint}/generate", + json={ + "model": self.model_name, + "prompt": prompt, + "temperature": temperature, + "max_tokens": max_tokens + }, + timeout=30 + ) + + response.raise_for_status() + data = response.json() + + latency = time.time() - start_time + + return { + 'text': data.get('text', ''), + 'tokens_used': data.get('tokens_used', 0), + 'latency': latency, + 'model': self.model_name + } + + except Exception as e: + self.logger.error(f"Local generation failed: {str(e)}") + raise + + def health_check(self): + """Check if local LLM is running.""" + try: + import requests + response = requests.get(f"{self.endpoint}/health", timeout=5) + response.raise_for_status() + return {'status': 'healthy', 'model': self.model_name} + except Exception as e: + return {'status': 'unhealthy', 'error': str(e)} + +class LLMOrchestrator: + """Orchestrate multiple LLM providers with fallback and load balancing.""" + + def __init__(self, providers): + self.providers = providers + self.current_provider = 0 + self.logger = logging.getLogger(__name__) + + def generate(self, prompt, **kwargs): + """Generate with automatic failover.""" + for attempt in range(len(self.providers)): + provider = self.providers[self.current_provider] + + try: + result = provider.generate(prompt, **kwargs) + result['provider'] = provider.__class__.__name__ + return result + + except Exception as e: + self.logger.warning(f"Provider {provider.__class__.__name__} failed: {str(e)}") + self.current_provider = (self.current_provider + 1) % len(self.providers) + + raise Exception("All LLM providers failed") + + def health_check(self): + """Health check for all providers.""" + health_status = {} + for i, provider in enumerate(self.providers): + health_status[f"provider_{i}"] = provider.health_check() + return health_status +``` + +### 2. Rate Limiting and Throttling +Implement rate limiting for API calls. + +```python +import threading +import time +from collections import deque + +class RateLimiter: + """Token bucket rate limiter.""" + + def __init__(self, rate, burst): + """ + Args: + rate: Tokens per second + burst: Maximum burst size + """ + self.rate = rate + self.burst = burst + self.tokens = burst + self.last_update = time.time() + self.lock = threading.Lock() + + def consume(self, tokens=1): + """Consume tokens if available.""" + with self.lock: + now = time.time() + elapsed = now - self.last_update + + # Refill tokens based on elapsed time + self.tokens = min(self.burst, self.tokens + elapsed * self.rate) + self.last_update = now + + if self.tokens >= tokens: + self.tokens -= tokens + return True + else: + return False + + def wait_for_token(self, tokens=1): + """Wait until tokens are available.""" + while not self.consume(tokens): + wait_time = (tokens - self.tokens) / self.rate + time.sleep(wait_time) + +class LLMWithRateLimit: + """LLM wrapper with rate limiting.""" + + def __init__(self, llm_provider, requests_per_second=10): + self.llm_provider = llm_provider + self.rate_limiter = RateLimiter(rate=requests_per_second, burst=20) + + def generate(self, prompt, **kwargs): + """Generate with rate limiting.""" + self.rate_limiter.wait_for_token() + return self.llm_provider.generate(prompt, **kwargs) + + def generate_batch(self, prompts, **kwargs): + """Generate multiple prompts with rate limiting.""" + results = [] + for prompt in prompts: + result = self.generate(prompt, **kwargs) + results.append(result) + return results +``` + +### 3. Caching and Response Management +Implement intelligent caching for LLM responses. + +```python +import hashlib +import json +from typing import Optional +import redis + +class LLMCache: + """Cache LLM responses.""" + + def __init__(self, redis_client=None, ttl=3600): + self.redis = redis_client + self.ttl = ttl + self.memory_cache = {} + + def _generate_cache_key(self, prompt, model, **kwargs): + """Generate cache key from prompt and parameters.""" + params = f"{prompt}:{model}:{json.dumps(kwargs, sort_keys=True)}" + return hashlib.sha256(params.encode()).hexdigest() + + def get(self, prompt, model, **kwargs) -> Optional[str]: + """Get cached response.""" + cache_key = self._generate_cache_key(prompt, model, **kwargs) + + # Check memory cache first + if cache_key in self.memory_cache: + return self.memory_cache[cache_key] + + # Check Redis + if self.redis: + cached = self.redis.get(cache_key) + if cached: + self.memory_cache[cache_key] = cached + return cached + + return None + + def set(self, prompt, response, model, **kwargs): + """Cache response.""" + cache_key = self._generate_cache_key(prompt, model, **kwargs) + + # Store in memory + self.memory_cache[cache_key] = response + + # Store in Redis + if self.redis: + self.redis.setex(cache_key, self.ttl, response) + +class CachedLLM: + """LLM with caching capability.""" + + def __init__(self, llm_provider, cache=None): + self.llm_provider = llm_provider + self.cache = cache or LLMCache() + + def generate(self, prompt, use_cache=True, **kwargs): + """Generate with optional caching.""" + model = getattr(self.llm_provider, 'model', 'unknown') + + if use_cache: + cached_response = self.cache.get(prompt, model, **kwargs) + if cached_response: + return { + 'text': cached_response, + 'cached': True, + 'model': model + } + + result = self.llm_provider.generate(prompt, **kwargs) + + if use_cache: + self.cache.set(prompt, result['text'], model, **kwargs) + + result['cached'] = False + return result +``` + +### 4. Monitoring and Metrics +Track LLM performance and usage. + +```python +from dataclasses import dataclass +from typing import List +import statistics + +@dataclass +class LLMCallMetrics: + """Metrics for individual LLM calls.""" + timestamp: float + model: str + tokens_used: int + latency: float + success: bool + error_message: str = "" + +class LLMMetricsCollector: + """Collect and analyze LLM metrics.""" + + def __init__(self, max_metrics=10000): + self.metrics: List[LLMCallMetrics] = [] + self.max_metrics = max_metrics + + def record_call(self, model, tokens_used, latency, success, error_message=""): + """Record metrics for an LLM call.""" + metric = LLMCallMetrics( + timestamp=time.time(), + model=model, + tokens_used=tokens_used, + latency=latency, + success=success, + error_message=error_message + ) + + self.metrics.append(metric) + + # Keep only recent metrics + if len(self.metrics) > self.max_metrics: + self.metrics = self.metrics[-self.max_metrics:] + + def get_statistics(self, model=None, time_window_seconds=None): + """Get statistics for LLM calls.""" + filtered_metrics = self.metrics + + if model: + filtered_metrics = [m for m in filtered_metrics if m.model == model] + + if time_window_seconds: + cutoff = time.time() - time_window_seconds + filtered_metrics = [m for m in filtered_metrics if m.timestamp > cutoff] + + if not filtered_metrics: + return {'error': 'No metrics found'} + + successful_calls = [m for m in filtered_metrics if m.success] + + stats = { + 'total_calls': len(filtered_metrics), + 'successful_calls': len(successful_calls), + 'error_rate': (len(filtered_metrics) - len(successful_calls)) / len(filtered_metrics), + 'avg_tokens': statistics.mean([m.tokens_used for m in successful_calls]) if successful_calls else 0, + 'avg_latency': statistics.mean([m.latency for m in successful_calls]) if successful_calls else 0, + 'p50_latency': statistics.median([m.latency for m in successful_calls]) if successful_calls else 0, + 'p95_latency': statistics.quantiles([m.latency for m in successful_calls], n=20)[18] if len(successful_calls) > 20 else 0, + 'total_tokens': sum([m.tokens_used for m in successful_calls]) + } + + return stats + +class InstrumentedLLM: + """LLM with automatic metrics collection.""" + + def __init__(self, llm_provider, metrics_collector): + self.llm_provider = llm_provider + self.metrics_collector = metrics_collector + + def generate(self, prompt, **kwargs): + """Generate with metrics collection.""" + model = getattr(self.llm_provider, 'model', 'unknown') + start_time = time.time() + + try: + result = self.llm_provider.generate(prompt, **kwargs) + latency = time.time() - start_time + + self.metrics_collector.record_call( + model=model, + tokens_used=result.get('tokens_used', 0), + latency=latency, + success=True + ) + + return result + + except Exception as e: + latency = time.time() - start_time + self.metrics_collector.record_call( + model=model, + tokens_used=0, + latency=latency, + success=False, + error_message=str(e) + ) + raise +``` + +### 5. Deployment Configuration +Manage deployment configurations. + +```python +from typing import Dict, Any +import yaml + +class LLMDeploymentConfig: + """Configuration for LLM deployment.""" + + def __init__(self, config_dict: Dict[str, Any]): + self.config = config_dict + + @classmethod + def from_file(cls, config_file: str): + """Load configuration from file.""" + with open(config_file, 'r') as f: + config_dict = yaml.safe_load(f) + return cls(config_dict) + + def get_provider_config(self, provider_name: str): + """Get configuration for specific provider.""" + return self.config.get('providers', {}).get(provider_name, {}) + + def get_model_config(self, model_name: str): + """Get configuration for specific model.""" + return self.config.get('models', {}).get(model_name, {}) + + def get_rate_limits(self) -> Dict[str, int]: + """Get rate limit configuration.""" + return self.config.get('rate_limits', { + 'requests_per_second': 10, + 'burst': 20 + }) + + def get_cache_config(self) -> Dict[str, Any]: + """Get cache configuration.""" + return self.config.get('cache', { + 'enabled': True, + 'ttl': 3600, + 'redis_url': None + }) + +# Example configuration file +example_config = { + 'providers': { + 'openai': { + 'api_key': 'your-api-key', + 'model': 'gpt-4', + 'temperature': 0.7, + 'max_tokens': 1000 + }, + 'local': { + 'endpoint': 'http://localhost:11434', + 'model': 'llama2', + 'temperature': 0.7 + } + }, + 'models': { + 'gpt-4': { + 'cost_per_1k_tokens': 0.03, + 'max_tokens': 8192 + }, + 'gpt-3.5-turbo': { + 'cost_per_1k_tokens': 0.002, + 'max_tokens': 4096 + } + }, + 'rate_limits': { + 'requests_per_second': 10, + 'burst': 20 + }, + 'cache': { + 'enabled': True, + 'ttl': 3600, + 'redis_url': 'redis://localhost:6379' + } +} +``` + +## Constraints +- **API Costs**: Monitor and control LLM API costs carefully +- **Latency**: LLM calls can be slow, implement proper timeouts +- **Rate Limits**: Respect provider rate limits to avoid being blocked +- **Error Handling**: Implement robust error handling and retry logic +- **Monitoring**: Track usage and performance for optimization +- **Scalability**: Design for horizontal scaling when needed + +## Expected Output +Production-ready LLM deployment with proper rate limiting, caching, monitoring, and multi-provider orchestration for reliable and cost-effective operations. diff --git a/TRAE-Skills/ai_engineering/LangChain_Basics.md b/TRAE-Skills/ai_engineering/LangChain_Basics.md new file mode 100644 index 0000000000000000000000000000000000000000..5558a9de69a316368ce659bf61436f6cef611734 --- /dev/null +++ b/TRAE-Skills/ai_engineering/LangChain_Basics.md @@ -0,0 +1,103 @@ +# Skill: LangChain Basics + +## Purpose +To utilize the LangChain framework to build complex LLM applications by chaining together components (Models, Prompts, Parsers) into composable workflows. + +## When to Use +- When building complex chains (e.g., Retrieval -> Augmentation -> Generation). +- When you need to swap LLM providers easily (e.g., OpenAI to Anthropic). +- When integrating structured output parsing. + +## Procedure + +### 1. Installation +Install core LangChain packages and the OpenAI integration. + +```bash +npm install @langchain/core @langchain/openai zod +``` + +### 2. Basic Chain Construction (LCEL) +Use LangChain Expression Language (LCEL) for declarative chain definitions. + +```typescript +import { ChatOpenAI } from "@langchain/openai"; +import { ChatPromptTemplate } from "@langchain/core/prompts"; +import { StringOutputParser } from "@langchain/core/output_parsers"; + +// 1. Initialize Model +const model = new ChatOpenAI({ + modelName: "gpt-4o", + temperature: 0, + apiKey: process.env.OPENAI_API_KEY +}); + +// 2. Define Prompt +const prompt = ChatPromptTemplate.fromMessages([ + ["system", "You are a technical documentation expert."], + ["user", "Explain {topic} in one sentence."] +]); + +// 3. Create Chain +// Input -> Prompt -> Model -> String Output +const chain = prompt.pipe(model).pipe(new StringOutputParser()); + +// Usage +async function runChain() { + const result = await chain.invoke({ topic: "Dependency Injection" }); + console.log(result); +} +``` + +### 3. Structured Output Parsing +Use `StructuredOutputParser` with Zod to guarantee type-safe responses. + +```typescript +import { z } from "zod"; +import { StructuredOutputParser } from "@langchain/core/output_parsers"; + +// Define Schema +const schema = z.object({ + sentiment: z.enum(["positive", "negative", "neutral"]), + keywords: z.array(z.string()).describe("List of up to 5 keywords"), + summary: z.string().describe("Brief summary of the text") +}); + +const parser = StructuredOutputParser.fromZodSchema(schema); + +const analysisChain = ChatPromptTemplate.fromTemplate( + "Analyze the following text.\n{format_instructions}\n\nText: {text}" +).pipe(model).pipe(parser); + +async function analyzeText(text: string) { + return await analysisChain.invoke({ + text, + format_instructions: parser.getFormatInstructions() + }); +} +``` + +### 4. Memory Integration (RunnableWithMessageHistory) +Manage conversation history for chatbots. + +```typescript +import { RunnableWithMessageHistory } from "@langchain/core/runnables"; +import { InMemoryChatMessageHistory } from "@langchain/core/chat_history"; + +const messageHistory = new InMemoryChatMessageHistory(); + +const chatChain = new RunnableWithMessageHistory({ + runnable: prompt.pipe(model), + getMessageHistory: async (sessionId) => messageHistory, + inputMessagesKey: "input", + historyMessagesKey: "history", +}); +``` + +## Constraints +- **Abstraction Cost**: LangChain adds a layer of abstraction. For very simple calls, the native SDK might be cleaner. +- **Debugging**: LCEL chains can be harder to debug than imperative code. Use `LangSmith` for tracing if available. +- **Version Compatibility**: LangChain evolves fast. Lock versions in `package.json`. + +## Expected Output +A composable pipeline that reliably transforms inputs into structured outputs, leveraging the power of chained LLM operations. diff --git a/TRAE-Skills/ai_engineering/Local_LLM_Running_Ollama.md b/TRAE-Skills/ai_engineering/Local_LLM_Running_Ollama.md new file mode 100644 index 0000000000000000000000000000000000000000..0a3ffee34f85dfe736ac4136a5e2034ad987ecac --- /dev/null +++ b/TRAE-Skills/ai_engineering/Local_LLM_Running_Ollama.md @@ -0,0 +1,94 @@ +# Skill: Local LLM Running (Ollama) + +## Purpose +To run Large Language Models locally using Ollama, ensuring data privacy, zero API costs, and offline capability while providing a standard REST API for applications. + +## When to Use +- When data privacy is paramount (medical, legal, personal data). +- For development and testing without incurring API costs. +- When you need to experiment with open-source models (Llama 3, Mistral, etc.). + +## Procedure + +### 1. Installation & Model Setup +Install Ollama and pull the desired model. + +```bash +# Install (macOS/Linux) +curl -fsSL https://ollama.com/install.sh | sh + +# Pull a model +ollama pull llama3:8b +``` + +### 2. Basic Usage (CLI) +Interact with the model directly in your terminal. + +```bash +ollama run llama3:8b "Why is the sky blue?" +``` + +### 3. Programmatic Integration (Node.js) +Use the Ollama REST API or official library to integrate into your app. + +```bash +npm install ollama +``` + +```typescript +import ollama from 'ollama'; + +async function chat() { + const response = await ollama.chat({ + model: 'llama3:8b', + messages: [{ role: 'user', content: 'Explain quantum physics to a 5-year old' }], + stream: true, + }); + + for await (const part of response) { + process.stdout.write(part.message.content); + } +} +``` + +### 4. Customizing Models (Modelfile) +Create a specialized version of a model with custom system prompts. + +1. Create a file named `Modelfile`: +```dockerfile +FROM llama3:8b + +# Set parameters +PARAMETER temperature 0.1 +PARAMETER top_p 0.9 + +# Set system message +SYSTEM """ +You are a senior TypeScript developer. +You provide concise, high-performance code snippets. +Always use ESM syntax. +""" +``` + +2. Create the model: +```bash +ollama create ts-expert -f Modelfile +``` + +### 5. Running as a Service (Docker) +Run Ollama in a container for consistent deployment. + +```bash +docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama +``` + +## Constraints +- **VRAM Requirements**: + - 7B/8B models: ~8GB RAM/VRAM. + - 13B models: ~16GB RAM/VRAM. + - 70B models: ~48GB+ RAM/VRAM. +- **Latency**: Local models are significantly slower than GPT-4o unless running on a high-end GPU (RTX 3090/4090 or Apple M2/M3 Max). +- **Quantization**: Most Ollama models are 4-bit quantized (Q4_K_M) by default, which slightly reduces reasoning capability but saves memory. + +## Expected Output +A locally running LLM service accessible via a REST API on `localhost:11434`. diff --git a/TRAE-Skills/ai_engineering/ML_Model_Quantization.md b/TRAE-Skills/ai_engineering/ML_Model_Quantization.md new file mode 100644 index 0000000000000000000000000000000000000000..6f0fe3e5a19dab349a208ec65e3c89bebd40e398 --- /dev/null +++ b/TRAE-Skills/ai_engineering/ML_Model_Quantization.md @@ -0,0 +1,85 @@ +# Skill: ML Model Quantization + +## Purpose +To reduce the memory footprint and improve inference speed of Machine Learning models, particularly deep neural networks, by converting high-precision weights (e.g., FP32) to lower-precision representations (e.g., INT8) with minimal loss of accuracy. + +## When to Use +- When deploying models to edge devices (mobile, IoT) with limited memory or compute +- When optimizing inference costs on cloud infrastructure +- When striving for real-time performance in computer vision or NLP tasks +- When the model size exceeds deployment constraints + +## Procedure + +### 1. Identify Quantization Strategy +Choose the appropriate quantization method based on your deployment needs: +- **Post-Training Quantization (PTQ)**: Applied after training. Easiest to implement. Good for general use cases. +- **Quantization-Aware Training (QAT)**: Simulates quantization during training. Results in higher accuracy, but requires retraining. + +### 2. Post-Training Quantization (PyTorch Example) + +**Dynamic Quantization** (Best for LSTM/RNN or Transformer models): +```python +import torch + +# 1. Load your pre-trained model +model = MyTransformerModel() +model.load_state_dict(torch.load('model_fp32.pth')) +model.eval() + +# 2. Apply dynamic quantization to specific layers (e.g., Linear layers) +quantized_model = torch.quantization.quantize_dynamic( + model, {torch.nn.Linear}, dtype=torch.qint8 +) + +# 3. Save the quantized model +torch.save(quantized_model.state_dict(), 'model_int8.pth') +``` + +**Static Quantization** (Best for CNNs): +Requires a representative dataset to calibrate the activations. +```python +import torch + +# 1. Prepare model for static quantization +model.eval() +model.qconfig = torch.quantization.get_default_qconfig('fbgemm') +torch.quantization.prepare(model, inplace=True) + +# 2. Calibrate with representative data +for data, _ in representative_dataloader: + model(data) + +# 3. Convert to quantized model +torch.quantization.convert(model, inplace=True) +``` + +### 3. Quantization-Aware Training (QAT) +If PTQ causes an unacceptable drop in accuracy, use QAT. + +```python +import torch + +# 1. Prepare model for QAT +model.train() +model.qconfig = torch.quantization.get_default_qat_qconfig('fbgemm') +torch.quantization.prepare_qat(model, inplace=True) + +# 2. Fine-tune the model +for epoch in range(num_epochs): + for data, target in train_dataloader: + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + optimizer.step() + +# 3. Convert to quantized model for inference +model.eval() +torch.quantization.convert(model, inplace=True) +``` + +## Best Practices +- Always benchmark accuracy before and after quantization. +- For LLMs, consider specialized quantization libraries like `bitsandbytes` (4-bit/8-bit) or formats like GGUF/AWQ. +- Use the appropriate backend (e.g., `fbgemm` for x86, `qnnpack` for ARM). \ No newline at end of file diff --git a/TRAE-Skills/ai_engineering/MultiModal_AI.md b/TRAE-Skills/ai_engineering/MultiModal_AI.md new file mode 100644 index 0000000000000000000000000000000000000000..516b5a177ac89f742dc79f1b11ddd3bb7f13b28a --- /dev/null +++ b/TRAE-Skills/ai_engineering/MultiModal_AI.md @@ -0,0 +1,333 @@ +# Skill: Multi-Modal AI + +## Purpose +To work with AI models that can process and generate multiple types of content including text, images, audio, and video in a unified framework. + +## When to Use +- When building applications that process images and text together +- When implementing vision-language tasks +- When generating images from text descriptions +- When analyzing visual content with natural language queries + +## Procedure + +### 1. Vision-Language Understanding +Process images with text queries using GPT-4 Vision. + +```python +from openai import OpenAI +import base64 + +client = OpenAI() + +def encode_image(image_path): + """Encode image to base64.""" + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode('utf-8') + +def analyze_image(image_path, question): + """Analyze image with text query.""" + base64_image = encode_image(image_path) + + response = client.chat.completions.create( + model="gpt-4-vision-preview", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": question}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64_image}" + } + } + ] + } + ], + max_tokens=500 + ) + + return response.choices[0].message.content + +# Example +analysis = analyze_image( + "product.jpg", + "What products are shown in this image? What are their key features?" +) +print(analysis) +``` + +### 2. Image Generation with DALL-E +Generate images from text descriptions. + +```python +def generate_image(prompt, size="1024x1024", quality="standard"): + """Generate image from text prompt.""" + response = client.images.generate( + model="dall-e-3", + prompt=prompt, + size=size, + quality=quality, + n=1 + ) + + image_url = response.data[0].url + revised_prompt = response.data[0].revised_prompt + + return { + "url": image_url, + "revised_prompt": revised_prompt + } + +# Example +result = generate_image( + "A futuristic smart home with voice-controlled devices and automated lighting", + size="1024x1024" +) +print(f"Image URL: {result['url']}") +``` + +### 3. Image Editing and Variations +Modify and create variations of existing images. + +```python +from PIL import Image +import io + +def edit_image(original_image_path, mask_path, prompt): + """Edit image with mask and prompt.""" + with open(original_image_path, "rb") as img_file: + original_image = img_file.read() + + with open(mask_path, "rb") as mask_file: + mask_image = mask_file.read() + + response = client.images.edit( + model="dall-e-2", + image=original_image, + mask=mask_image, + prompt=prompt, + n=1, + size="512x512" + ) + + return response.data[0].url + +def create_variations(image_path): + """Create variations of an image.""" + with open(image_path, "rb") as img_file: + image_data = img_file.read() + + response = client.images.create_variation( + image=image_data, + n=3, + size="1024x1024" + ) + + return [img.url for img in response.data] + +# Example +edited_url = edit_image( + "room.jpg", + "room_mask.png", + "Add a modern desk with computer setup" +) +print(f"Edited image: {edited_url}") +``` + +### 4. Multi-Modal Document Processing +Process documents with text and images. + +```python +import pdf2image +import pytesseract + +def process_multimodal_document(pdf_path): + """Extract and analyze content from PDF with images.""" + # Convert PDF to images + images = pdf2image.convert_from_path(pdf_path) + + results = [] + + for i, image in enumerate(images): + # Extract text using OCR + text = pytesseract.image_to_string(image) + + # Encode image for GPT-4 Vision analysis + import io + import base64 + + img_byte_arr = io.BytesIO() + image.save(img_byte_arr, format='PNG') + img_byte_arr = img_byte_arr.getvalue() + base64_image = base64.b64encode(img_byte_arr).decode('utf-8') + + # Analyze image content + vision_response = client.chat.completions.create( + model="gpt-4-vision-preview", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Analyze this document page. What type of content is shown?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{base64_image}"} + } + ] + } + ] + ) + + results.append({ + "page": i + 1, + "ocr_text": text, + "visual_analysis": vision_response.choices[0].message.content + }) + + return results + +# Example +document_results = process_multimodal_document("contract.pdf") +for page in document_results: + print(f"Page {page['page']}: {page['visual_analysis']}") +``` + +### 5. Audio and Text Integration +Process audio with text analysis. + +```python +import requests + +def transcribe_and_analyze(audio_path): + """Transcribe audio and analyze with text.""" + # Transcribe audio using Whisper + with open(audio_path, "rb") as audio_file: + transcription_response = client.audio.transcriptions.create( + model="whisper-1", + file=audio_file, + response_format="text" + ) + + transcript = transcription_response + + # Analyze transcript with GPT-4 + analysis = client.chat.completions.create( + model="gpt-4", + messages=[{ + "role": "user", + "content": f"Analyze this transcript for key topics, sentiment, and action items:\n\n{transcript}" + }] + ) + + return { + "transcript": transcript, + "analysis": analysis.choices[0].message.content + } + +# Example +result = transcribe_and_analyze("meeting_recording.mp3") +print(f"Transcript: {result['transcript']}") +print(f"Analysis: {result['analysis']}") +``` + +### 6. Multi-Modal RAG System +Build RAG with image and text retrieval. + +```python +from sentence_transformers import SentenceTransformer +import faiss +import numpy as np + +class MultiModalRAG: + def __init__(self): + self.text_model = SentenceTransformer('all-MiniLM-L6-v2') + self.image_model = SentenceTransformer('clip-ViT-B-32') + self.text_index = None + self.image_index = None + self.text_docs = [] + self.image_docs = [] + + def add_text_documents(self, documents): + """Add text documents to the index.""" + embeddings = self.text_model.encode(documents) + + if self.text_index is None: + dimension = embeddings.shape[1] + self.text_index = faiss.IndexFlatL2(dimension) + + self.text_index.add(embeddings.astype('float32')) + self.text_docs.extend(documents) + + def add_image_documents(self, image_paths, descriptions): + """Add images to the index.""" + embeddings = self.image_model.encode(image_paths) + + if self.image_index is None: + dimension = embeddings.shape[1] + self.image_index = faiss.IndexFlatL2(dimension) + + self.image_index.add(embeddings.astype('float32')) + self.image_docs.extend(zip(image_paths, descriptions)) + + def search(self, query, k=3): + """Search across text and images.""" + # Search text + if self.text_index: + query_embedding = self.text_model.encode([query]) + text_distances, text_indices = self.text_index.search( + query_embedding.astype('float32'), k + ) + text_results = [ + (self.text_docs[i], text_distances[0][j]) + for j, i in enumerate(text_indices[0]) + ] + else: + text_results = [] + + # Search images + if self.image_index: + query_embedding = self.image_model.encode([query]) + image_distances, image_indices = self.image_index.search( + query_embedding.astype('float32'), k + ) + image_results = [ + (self.image_docs[i], image_distances[0][j]) + for j, i in enumerate(image_indices[0]) + ] + else: + image_results = [] + + return { + "text_results": text_results, + "image_results": image_results + } + +# Example usage +rag = MultiModalRAG() + +rag.add_text_documents([ + "Python is a high-level programming language", + "JavaScript is used for web development" +]) + +rag.add_image_documents( + ["python_logo.png", "js_logo.png"], + ["Python programming language logo", "JavaScript logo"] +) + +results = rag.search("programming languages") +print(results) +``` + +## Constraints +- **Image Size**: Vision models have limits on image dimensions and file sizes +- **Cost**: Vision and image generation APIs are more expensive than text-only +- **Quality**: Generated images may not always match expectations +- **Processing Time**: Image processing is slower than text-only operations +- **Accuracy**: OCR accuracy varies based on image quality +- **Privacy**: Be careful with sensitive visual content + +## Expected Output +Comprehensive multi-modal AI applications that can seamlessly process and generate text, images, and audio content with high accuracy and integration. diff --git a/TRAE-Skills/ai_engineering/Natural_Language_to_SQL.md b/TRAE-Skills/ai_engineering/Natural_Language_to_SQL.md new file mode 100644 index 0000000000000000000000000000000000000000..895ebe18f8d2232ab6f6876e9df5c50e146d8449 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Natural_Language_to_SQL.md @@ -0,0 +1,152 @@ +# Skill: Natural Language to SQL (NL2SQL) + +## Purpose +To build systems that convert natural language questions into executable SQL queries, enabling non-technical users to interact with databases. + +## When to Use +- When building analytics dashboards for business users +- For customer support tools that query databases directly +- When implementing internal tools for non-technical teams +- For data exploration applications +- When building chatbots that need database access + +## Procedure + +### 1. Simple NL2SQL with OpenAI +Use LLMs to generate SQL from natural language. + +```python +import openai +import sqlite3 + +client = openai.OpenAI(api_key="your-api-key") + +def nl_to_sql(question, table_schema): + prompt = f"""Given the following table schema: +{table_schema} + +Generate a SQLite SQL query to answer this question: {question} + +Only return the SQL query, no explanation. +""" + + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}] + ) + + return response.choices[0].message.content.strip() + +# Example usage +table_schema = """ +Table: users +- id: INTEGER +- name: TEXT +- email: TEXT +- created_at: DATE +- status: TEXT (active, inactive) + +Table: orders +- id: INTEGER +- user_id: INTEGER +- total_amount: DECIMAL +- order_date: DATE +- status: TEXT (pending, completed, cancelled) +""" + +question = "Show me all active users who placed orders over $100 in 2024" +sql_query = nl_to_sql(question, table_schema) +print(sql_query) +``` + +### 2. Execute Query Safely +Execute the generated SQL with safety checks. + +```python +def execute_safe_query(db_path, sql): + # Safety checks + dangerous_keywords = ['DROP', 'DELETE', 'TRUNCATE', 'ALTER', 'INSERT', 'UPDATE', 'CREATE'] + for keyword in dangerous_keywords: + if keyword.upper() in sql.upper(): + raise Exception(f"Query contains forbidden operation: {keyword}") + + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute(sql) + results = cursor.fetchall() + columns = [description[0] for description in cursor.description] + conn.close() + + return {"columns": columns, "results": results} + +# Usage +try: + result = execute_safe_query("mydb.sqlite", sql_query) + print("Columns:", result["columns"]) + print("Results:", result["results"]) +except Exception as e: + print("Error:", e) +``` + +### 3. Few-Shot Learning Examples +Improve accuracy with few-shot examples. + +```python +few_shot_examples = """ +Example 1: +Question: How many users are there? +SQL: SELECT COUNT(*) FROM users; + +Example 2: +Question: Show users who signed up in 2024 +SQL: SELECT * FROM users WHERE created_at >= '2024-01-01'; + +Example 3: +Question: What's the total revenue from completed orders? +SQL: SELECT SUM(total_amount) FROM orders WHERE status = 'completed'; +""" + +def nl_to_sql_with_examples(question, table_schema): + prompt = f"""Given the following table schema: +{table_schema} + +Examples: +{few_shot_examples} + +Generate a SQLite SQL query to answer this question: {question} +Only return the SQL query. +""" + + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}] + ) + + return response.choices[0].message.content.strip() +``` + +### 4. Using LangChain for NL2SQL +Use LangChain's SQL database chain. + +```python +from langchain_openai import ChatOpenAI +from langchain_community.utilities import SQLDatabase +from langchain.chains import create_sql_query_chain + +db = SQLDatabase.from_uri("sqlite:///mydb.sqlite") +llm = ChatOpenAI(model="gpt-4", temperature=0) +chain = create_sql_query_chain(llm, db) + +question = "Show me the top 5 users by total order amount" +response = chain.invoke({"question": question}) +print(response) +``` + +## Best Practices +- **Whitelist Operations**: Only allow SELECT queries in production +- **Schema Context**: Always provide clear table schema information +- **Validation**: Validate generated queries before execution +- **Few-Shot Learning**: Use examples to improve accuracy +- **Error Handling**: Gracefully handle query generation failures +- **Sanitize Inputs**: Prevent SQL injection even from generated queries +- **Log Everything**: Log questions, queries, and results for debugging diff --git a/TRAE-Skills/ai_engineering/OpenAI_API_Integration.md b/TRAE-Skills/ai_engineering/OpenAI_API_Integration.md new file mode 100644 index 0000000000000000000000000000000000000000..baaa392f67a11b8da3aa42722bb733a3709b009c --- /dev/null +++ b/TRAE-Skills/ai_engineering/OpenAI_API_Integration.md @@ -0,0 +1,142 @@ +# Skill: OpenAI API Integration + +## Purpose +To correctly integrate OpenAI's models (GPT-4o, GPT-3.5) into Node.js/TypeScript applications, handling authentication, type-safety, streaming, and robust error management. + +## When to Use +- When building applications that rely on OpenAI's completion or chat API. +- When you need a production-ready wrapper around the official SDK. + +## Procedure + +### 1. Installation & Setup +Install the official library and type definitions. + +```bash +npm install openai zod +``` + +### 2. Robust Client Implementation +Create a service class that handles initialization and configuration. + +```typescript +// lib/openai-client.ts +import OpenAI from 'openai'; + +export class OpenAIClient { + private client: OpenAI; + + constructor() { + if (!process.env.OPENAI_API_KEY) { + throw new Error("Missing OPENAI_API_KEY environment variable"); + } + this.client = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, + maxRetries: 3, // Built-in retry logic + timeout: 30000, // 30s timeout + }); + } + + public getClient() { + return this.client; + } +} +``` + +### 3. Chat Completion with Error Handling +Implement a method for standard chat completions with try/catch blocks for specific API errors. + +```typescript +// lib/ai-service.ts +import { OpenAIClient } from './openai-client'; +import OpenAI from 'openai'; + +export class AIService { + private openai: OpenAI; + + constructor() { + this.openai = new OpenAIClient().getClient(); + } + + async generateResponse(systemPrompt: string, userMessage: string): Promise { + try { + const response = await this.openai.chat.completions.create({ + model: "gpt-4o", + messages: [ + { role: "system", content: systemPrompt }, + { role: "user", content: userMessage }, + ], + temperature: 0.7, + }); + + return response.choices[0]?.message?.content || ""; + } catch (error) { + if (error instanceof OpenAI.APIError) { + console.error(`OpenAI Error: ${error.status} - ${error.code}`); + // Handle specific codes: 429 (Rate Limit), 400 (Bad Request), 401 (Auth) + if (error.status === 429) { + throw new Error("Rate limit exceeded. Please try again later."); + } + } + throw new Error("Failed to generate AI response"); + } + } +} +``` + +### 4. Streaming Response Implementation +Handle real-time output for better UX. + +```typescript +// lib/stream-service.ts +import { OpenAIClient } from './openai-client'; + +export async function* streamCompletion(prompt: string) { + const openai = new OpenAIClient().getClient(); + + const stream = await openai.chat.completions.create({ + model: "gpt-4o", + messages: [{ role: "user", content: prompt }], + stream: true, + }); + + for await (const chunk of stream) { + const content = chunk.choices[0]?.delta?.content || ""; + if (content) { + yield content; + } + } +} + +// Usage: +// for await (const token of streamCompletion("Hello")) { +// process.stdout.write(token); +// } +``` + +### 5. Structured Outputs (JSON) +Enforce JSON output for programmatic use. + +```typescript +async function extractData(text: string) { + const openai = new OpenAIClient().getClient(); + const response = await openai.chat.completions.create({ + model: "gpt-4o", + messages: [ + { role: "system", content: "You are a data extractor. Output valid JSON." }, + { role: "user", content: `Extract names from: ${text}` } + ], + response_format: { type: "json_object" }, + }); + + return JSON.parse(response.choices[0].message.content!); +} +``` + +## Constraints +- **Costs**: GPT-4 is expensive. Cache responses where possible (e.g., using Redis) for identical inputs. +- **Security**: Never expose the API key on the client-side (browser). Always proxy requests through your backend. +- **Timeouts**: LLM requests can be slow. Ensure your HTTP server (e.g., Nginx, Vercel) has appropriate timeout settings (often > 10s). + +## Expected Output +A secure, reusable service module that reliably communicates with OpenAI, handles rate limits, and provides both streaming and blocking interfaces. diff --git a/TRAE-Skills/ai_engineering/Prompt_Engineering_Basics.md b/TRAE-Skills/ai_engineering/Prompt_Engineering_Basics.md new file mode 100644 index 0000000000000000000000000000000000000000..5c86dfeb2766a559aeb1bedc67bf1160eb610bbe --- /dev/null +++ b/TRAE-Skills/ai_engineering/Prompt_Engineering_Basics.md @@ -0,0 +1,130 @@ +# Skill: Prompt Engineering Basics + +## Purpose +To apply systematic techniques for crafting inputs (prompts) to Large Language Models (LLMs) to ensure accurate, consistent, and high-quality outputs. This skill focuses on structured prompt management and engineering patterns. + +## When to Use +- When interacting with LLMs for any task (generation, summarization, extraction). +- When model outputs are inconsistent, hallucinatory, or vague. +- When you need to standardize input/output formats for an application. + +## Procedure + +### 1. Structured Prompt Management (TypeScript) +Avoid hardcoding strings. Use a template system. + +```typescript +// prompt-manager.ts +type PromptVariables = Record; + +interface PromptTemplate { + system: (vars: PromptVariables) => string; + user: (vars: PromptVariables) => string; +} + +export const summarizationPrompt: PromptTemplate = { + system: ({ context }) => + `You are an expert technical writer. Your task is to summarize the following context into a concise executive summary. + + Context: ${context} + + Constraints: + - Use bullet points. + - Max 200 words. + - Maintain a professional tone.`, + + user: ({ text }) => `Text to summarize:\n${text}` +}; +``` + +### 2. The CO-STAR Framework Implementation +Implement the CO-STAR framework programmatically to ensure all context is provided. + +```typescript +// co-star-prompt.ts +interface CoStarParams { + context: string; + objective: string; + style: string; + tone: string; + audience: string; + responseFormat: string; +} + +export const buildCoStarSystemMessage = (params: CoStarParams): string => { + return ` + # CONTEXT + ${params.context} + + # OBJECTIVE + ${params.objective} + + # STYLE + ${params.style} + + # TONE + ${params.tone} + + # AUDIENCE + ${params.audience} + + # RESPONSE FORMAT + ${params.responseFormat} + `; +}; +``` + +### 3. Chain-of-Thought (CoT) & Few-Shot Prompting +Enhance reasoning by forcing step-by-step logic or providing examples. + +```typescript +// reasoning-prompt.ts +const fewShotExamples = ` +Input: "The server is down." +Classification: Critical +Reasoning: Impact on availability is immediate. + +Input: "The button color is slightly off." +Classification: Low +Reasoning: Purely cosmetic issue. +`; + +export const classificationPrompt = (input: string) => ` +Classify the severity of the following issue (Critical, High, Medium, Low). +First, explain your reasoning step-by-step, then provide the final classification. + +Examples: +${fewShotExamples} + +Input: "${input}" +Output: +`; +``` + +### 4. Handling Output Formats (JSON Mode) +Always enforce structure when programmatic consumption is needed. + +```typescript +// Ensure your API call sets { response_format: { type: "json_object" } } +export const jsonExtractionPrompt = (text: string) => ` +Extract the key entities from the text below. +You must respond with valid JSON only. + +Schema: +{ + "names": string[], + "dates": string[], + "sentiment": "positive" | "negative" | "neutral" +} + +Text: "${text}" +`; +``` + +## Constraints +- **Context Window**: Monitor token count. Truncate inputs if they exceed limits (e.g., using `tiktoken`). +- **Injection Attacks**: Treat user input as untrusted. Delimit user input (e.g., with `"""` or `###`) to prevent prompt injection. +- **Determinism**: Set `temperature: 0` for classification/extraction tasks; higher for creative tasks. + +## Expected Output +A set of typed, reusable prompt templates that produce consistent, parsed outputs from the LLM. diff --git a/TRAE-Skills/ai_engineering/RAG_System_Architecture.md b/TRAE-Skills/ai_engineering/RAG_System_Architecture.md new file mode 100644 index 0000000000000000000000000000000000000000..48fae509d9df48f348abd5223982ab0d9416f98e --- /dev/null +++ b/TRAE-Skills/ai_engineering/RAG_System_Architecture.md @@ -0,0 +1,110 @@ +# Skill: RAG System Architecture + +## Purpose +To design and implement a Retrieval-Augmented Generation (RAG) system that grounds LLM responses in specific, external data sources, reducing hallucinations and enabling knowledge on private data. + +## When to Use +- When the LLM needs to answer questions about proprietary documents (PDFs, internal wikis). +- When the knowledge base is too large to fit in the context window. +- When data changes frequently and retraining is not feasible. + +## Procedure + +### 1. Ingestion & Embedding (The ETL Pipeline) +Process documents into vectors. + +```typescript +import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; +import { OpenAIEmbeddings } from "@langchain/openai"; +import { MemoryVectorStore } from "langchain/vectorstores/memory"; + +// 1. Text Splitting +const splitter = new RecursiveCharacterTextSplitter({ + chunkSize: 1000, + chunkOverlap: 200, // Crucial for context continuity +}); + +// 2. Embedding Model +const embeddings = new OpenAIEmbeddings({ + modelName: "text-embedding-3-small", // Efficient & cheap +}); + +// 3. Vector Store Initialization +// In production, use Pinecone, Weaviate, or pgvector +const vectorStore = new MemoryVectorStore(embeddings); + +export async function ingestDocument(text: string) { + const docs = await splitter.createDocuments([text]); + await vectorStore.addDocuments(docs); + return vectorStore; +} +``` + +### 2. Retrieval Implementation +Create a retriever that fetches relevant context. + +```typescript +// Create a retriever from the store +const retriever = vectorStore.asRetriever({ + k: 3, // Top 3 results + searchType: "similarity", // or "mmr" for diversity +}); +``` + +### 3. The RAG Chain (Generation) +Combine retrieval with generation. + +```typescript +import { ChatOpenAI } from "@langchain/openai"; +import { ChatPromptTemplate } from "@langchain/core/prompts"; +import { createStuffDocumentsChain } from "langchain/chains/combine_documents"; +import { createRetrievalChain } from "langchain/chains/retrieval"; + +const model = new ChatOpenAI({ modelName: "gpt-4o" }); + +const prompt = ChatPromptTemplate.fromTemplate(` +Answer the user's question based ONLY on the following context: + + +{context} + + +Question: {input} +`); + +// 1. Create the document combining chain +const combineDocsChain = await createStuffDocumentsChain({ + llm: model, + prompt, +}); + +// 2. Create the full retrieval chain +const ragChain = await createRetrievalChain({ + retriever, + combineDocsChain, +}); + +// Usage +export async function askQuestion(question: string) { + const response = await ragChain.invoke({ + input: question, + }); + return response.answer; +} +``` + +### 4. Advanced: Hybrid Search +For production, combine keyword search (BM25) with semantic search (Vectors) for better accuracy. + +```typescript +// Conceptual example for Supabase/Postgres +// supabase.rpc('hybrid_search', { query_text: ..., match_threshold: ... }) +``` + +## Constraints +- **Chunk Quality**: If chunks are too small, context is lost. If too large, noise increases. +- **Latency**: Embedding + Vector Search + Generation = High Latency. Use caching and streaming. +- **Relevance**: "Garbage In, Garbage Out". Ensure the retrieved context is actually relevant before passing to LLM. + +## Expected Output +A functional RAG pipeline where a user asks a question, the system retrieves relevant docs, and the LLM answers accurately citing the sources. diff --git a/TRAE-Skills/ai_engineering/Recommender_Systems_Collaborative_Filtering.md b/TRAE-Skills/ai_engineering/Recommender_Systems_Collaborative_Filtering.md new file mode 100644 index 0000000000000000000000000000000000000000..d6f40225dcfe753c83948340db77035e7d540850 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Recommender_Systems_Collaborative_Filtering.md @@ -0,0 +1,198 @@ +# Skill: Recommender Systems with Collaborative Filtering + +## Purpose +To build recommendation systems using collaborative filtering techniques for personalized user experiences. + +## When to Use +- When building product recommendations for e-commerce +- For movie/music/content recommendations +- When personalizing user experiences +- For "Customers who bought this also bought" features +- When you have user-item interaction data + +## Procedure + +### 1. User-Based Collaborative Filtering +Recommend items based on similar users. + +```python +import numpy as np +import pandas as pd +from sklearn.metrics.pairwise import cosine_similarity + +# Sample user-item ratings matrix +ratings_data = { + 'User1': [5, 4, 0, 0, 3], + 'User2': [0, 5, 4, 0, 0], + 'User3': [4, 0, 5, 3, 0], + 'User4': [0, 0, 4, 5, 4], + 'User5': [3, 0, 0, 4, 5] +} +items = ['ItemA', 'ItemB', 'ItemC', 'ItemD', 'ItemE'] + +ratings_matrix = pd.DataFrame(ratings_data, index=items).T + +def user_based_cf(user_id, ratings_matrix, n_recommendations=3): + # Calculate user similarity + user_similarity = cosine_similarity(ratings_matrix) + user_similarity_df = pd.DataFrame(user_similarity, index=ratings_matrix.index, columns=ratings_matrix.index) + + # Get similar users + similar_users = user_similarity_df[user_id].sort_values(ascending=False)[1:] + + # Predict ratings + user_ratings = ratings_matrix.loc[user_id] + predicted_ratings = pd.Series(dtype='float64') + + for item in ratings_matrix.columns: + if user_ratings[item] == 0: + weighted_sum = 0 + similarity_sum = 0 + for similar_user in similar_users.index: + if ratings_matrix.loc[similar_user, item] > 0: + weighted_sum += similar_users[similar_user] * ratings_matrix.loc[similar_user, item] + similarity_sum += similar_users[similar_user] + if similarity_sum > 0: + predicted_ratings[item] = weighted_sum / similarity_sum + + return predicted_ratings.sort_values(ascending=False).head(n_recommendations) + +# Usage +recommendations = user_based_cf('User1', ratings_matrix) +print("Recommendations for User1:", recommendations) +``` + +### 2. Item-Based Collaborative Filtering +Recommend items similar to those the user liked. + +```python +def item_based_cf(user_id, ratings_matrix, n_recommendations=3): + # Calculate item similarity + item_similarity = cosine_similarity(ratings_matrix.T) + item_similarity_df = pd.DataFrame(item_similarity, index=ratings_matrix.columns, columns=ratings_matrix.columns) + + user_ratings = ratings_matrix.loc[user_id] + predicted_ratings = pd.Series(dtype='float64') + + for item in ratings_matrix.columns: + if user_ratings[item] == 0: + weighted_sum = 0 + similarity_sum = 0 + for rated_item in user_ratings.index: + if user_ratings[rated_item] > 0: + weighted_sum += item_similarity_df[item][rated_item] * user_ratings[rated_item] + similarity_sum += item_similarity_df[item][rated_item] + if similarity_sum > 0: + predicted_ratings[item] = weighted_sum / similarity_sum + + return predicted_ratings.sort_values(ascending=False).head(n_recommendations) +``` + +### 3. Matrix Factorization with SVD +Use Singular Value Decomposition for better recommendations. + +```python +from scipy.sparse.linalg import svds + +def svd_recommender(ratings_matrix, user_id, n_recommendations=3, n_factors=2): + # Convert to numpy array and center + ratings = ratings_matrix.values + user_ratings_mean = np.mean(ratings, axis=1) + ratings_demeaned = ratings - user_ratings_mean.reshape(-1, 1) + + # Perform SVD + U, sigma, Vt = svds(ratings_demeaned, k=n_factors) + sigma = np.diag(sigma) + + # Reconstruct ratings + predicted_ratings = np.dot(np.dot(U, sigma), Vt) + user_ratings_mean.reshape(-1, 1) + predicted_ratings_df = pd.DataFrame(predicted_ratings, index=ratings_matrix.index, columns=ratings_matrix.columns) + + # Get recommendations + user_ratings = ratings_matrix.loc[user_id] + recommendations = predicted_ratings_df.loc[user_id][user_ratings == 0].sort_values(ascending=False).head(n_recommendations) + + return recommendations +``` + +### 4. Using Surprise Library +Use the Surprise library for recommendation systems. + +```python +from surprise import Dataset, Reader, SVD, KNNBasic +from surprise.model_selection import train_test_split +from surprise.metrics import accuracy + +# Load data +ratings_dict = { + 'userID': [1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4], + 'itemID': [1, 2, 3, 1, 2, 4, 2, 3, 4, 1, 3, 4], + 'rating': [5, 4, 3, 5, 4, 4, 4, 5, 5, 3, 5, 4] +} + +reader = Reader(rating_scale=(1, 5)) +data = Dataset.load_from_df(pd.DataFrame(ratings_dict)[['userID', 'itemID', 'rating']], reader) + +# Split data +trainset, testset = train_test_split(data, test_size=0.25) + +# Train SVD model +algo = SVD() +algo.fit(trainset) + +# Test +predictions = algo.test(testset) +print("RMSE:", accuracy.rmse(predictions)) + +# Get recommendations for a user +def get_surprise_recommendations(algo, user_id, item_ids, n_recommendations=3): + predictions = [] + for item_id in item_ids: + pred = algo.predict(user_id, item_id) + predictions.append((item_id, pred.est)) + + predictions.sort(key=lambda x: x[1], reverse=True) + return predictions[:n_recommendations] +``` + +### 5. Hybrid Recommender +Combine collaborative filtering with content-based filtering. + +```python +def hybrid_recommender(user_id, ratings_matrix, item_features, n_recommendations=3): + # Get collaborative filtering recommendations + cf_recs = user_based_cf(user_id, ratings_matrix, n_recommendations=5) + + # Get content-based recommendations (using item features) + item_similarity = cosine_similarity(item_features) + user_rated_items = ratings_matrix.loc[user_id][ratings_matrix.loc[user_id] > 0].index + + content_recs = pd.Series(dtype='float64') + for item in ratings_matrix.columns: + if item not in user_rated_items: + sim_sum = 0 + for rated_item in user_rated_items: + sim_sum += item_similarity[list(ratings_matrix.columns).index(item)][list(ratings_matrix.columns).index(rated_item)] + content_recs[item] = sim_sum / len(user_rated_items) + + content_recs = content_recs.sort_values(ascending=False).head(5) + + # Combine recommendations (simple average) + combined_recs = pd.Series(dtype='float64') + for item in set(cf_recs.index).union(set(content_recs.index)): + cf_score = cf_recs.get(item, 0) + content_score = content_recs.get(item, 0) + combined_recs[item] = (cf_score + content_score) / 2 + + return combined_recs.sort_values(ascending=False).head(n_recommendations) +``` + +## Best Practices +- **Data Preprocessing**: Clean and preprocess your data thoroughly +- **Cold Start**: Handle new users/items with hybrid approaches +- **Evaluation**: Use RMSE, MAE, or ranking metrics (NDCG, MAP) +- **Scalability**: Use matrix factorization or deep learning for large datasets +- **Diversity**: Ensure recommendations are diverse, not just similar +- **Freshness**: Update recommendations regularly with new data +- **A/B Testing**: Always test recommendations with real users +- **Privacy**: Be mindful of user privacy and data usage diff --git a/TRAE-Skills/ai_engineering/Reinforcement_Learning_Basics.md b/TRAE-Skills/ai_engineering/Reinforcement_Learning_Basics.md new file mode 100644 index 0000000000000000000000000000000000000000..343c1a4be7de721fad2551a951b0451d2436fb3b --- /dev/null +++ b/TRAE-Skills/ai_engineering/Reinforcement_Learning_Basics.md @@ -0,0 +1,335 @@ +# Skill: Reinforcement Learning Basics + +## Purpose +To implement intelligent agents that learn optimal behaviors through interaction with an environment, using rewards and penalties to guide decision-making. + +## When to Use +- When building game-playing AI +- When optimizing control systems (robotics, autonomous vehicles) +- When solving sequential decision problems +- When implementing recommendation systems with user feedback + +## Procedure + +### 1. Q-Learning Implementation +Implement basic Q-learning algorithm. + +```python +import numpy as np +from collections import defaultdict + +class QLearningAgent: + def __init__(self, state_size, action_size, learning_rate=0.1, discount_factor=0.95, epsilon=1.0): + self.state_size = state_size + self.action_size = action_size + self.learning_rate = learning_rate + self.discount_factor = discount_factor + self.epsilon = epsilon + self.epsilon_decay = 0.995 + self.epsilon_min = 0.01 + self.q_table = defaultdict(lambda: np.zeros(action_size)) + + def choose_action(self, state): + """Choose action using epsilon-greedy policy.""" + if np.random.random() <= self.epsilon: + return np.random.choice(self.action_size) + else: + return np.argmax(self.q_table[state]) + + def learn(self, state, action, reward, next_state, done): + """Update Q-value using Q-learning formula.""" + current_q = self.q_table[state][action] + + if done: + max_next_q = 0 + else: + max_next_q = np.max(self.q_table[next_state]) + + new_q = current_q + self.learning_rate * (reward + self.discount_factor * max_next_q - current_q) + self.q_table[state][action] = new_q + + # Decay epsilon + if self.epsilon > self.epsilon_min: + self.epsilon *= self.epsilon_decay + + def save_q_table(self, filename): + """Save Q-table to file.""" + dict_q_table = dict(self.q_table) + np.save(filename, dict_q_table) + + def load_q_table(self, filename): + """Load Q-table from file.""" + dict_q_table = np.load(filename, allow_pickle=True).item() + self.q_table = defaultdict(lambda: np.zeros(self.action_size), dict_q_table) + +# Simple grid world environment +class GridWorld: + def __init__(self, size=5): + self.size = size + self.start = (0, 0) + self.goal = (size-1, size-1) + self.obstacles = [(2, 2), (3, 1), (1, 3)] + self.current_state = self.start + self.actions = ['up', 'down', 'left', 'right'] + + def reset(self): + self.current_state = self.start + return self.current_state + + def step(self, action): + row, col = self.current_state + + if action == 'up': + new_row, new_col = row - 1, col + elif action == 'down': + new_row, new_col = row + 1, col + elif action == 'left': + new_row, new_col = row, col - 1 + elif action == 'right': + new_row, new_col = row, col + 1 + + # Check boundaries + if 0 <= new_row < self.size and 0 <= new_col < self.size: + if (new_row, new_col) not in self.obstacles: + self.current_state = (new_row, new_col) + + # Calculate reward + if self.current_state == self.goal: + reward = 10 + done = True + else: + reward = -1 + done = False + + return self.current_state, reward, done + +# Training +env = GridWorld() +agent = QLearningAgent(state_size=env.size*env.size, action_size=4) + +episodes = 1000 +for episode in range(episodes): + state = env.reset() + done = False + total_reward = 0 + + while not done: + action_idx = agent.choose_action(state) + action = env.actions[action_idx] + next_state, reward, done = env.step(action) + + agent.learn(state, action_idx, reward, next_state, done) + state = next_state + total_reward += reward + + if episode % 100 == 0: + print(f"Episode {episode}, Total Reward: {total_reward}, Epsilon: {agent.epsilon:.3f}") +``` + +### 2. Deep Q-Network (DQN) +Implement deep Q-learning with neural networks. + +```python +import torch +import torch.nn as nn +import torch.optim as optim +from collections import deque +import random + +class DQN(nn.Module): + def __init__(self, state_size, action_size): + super(DQN, self).__init__() + self.fc1 = nn.Linear(state_size, 64) + self.fc2 = nn.Linear(64, 64) + self.fc3 = nn.Linear(64, action_size) + + def forward(self, x): + x = torch.relu(self.fc1(x)) + x = torch.relu(self.fc2(x)) + return self.fc3(x) + +class DQNAgent: + def __init__(self, state_size, action_size): + self.state_size = state_size + self.action_size = action_size + self.memory = deque(maxlen=10000) + self.gamma = 0.95 + self.epsilon = 1.0 + self.epsilon_min = 0.01 + self.epsilon_decay = 0.995 + self.learning_rate = 0.001 + self.batch_size = 32 + + self.model = DQN(state_size, action_size) + self.optimizer = optim.Adam(self.model.parameters(), lr=self.learning_rate) + + def remember(self, state, action, reward, next_state, done): + """Store experience in replay memory.""" + self.memory.append((state, action, reward, next_state, done)) + + def act(self, state): + """Choose action using epsilon-greedy policy.""" + if np.random.random() <= self.epsilon: + return random.randrange(self.action_size) + + state = torch.FloatTensor(state).unsqueeze(0) + q_values = self.model(state) + return np.argmax(q_values.detach().numpy()[0]) + + def replay(self): + """Train on a batch of experiences.""" + if len(self.memory) < self.batch_size: + return + + minibatch = random.sample(self.memory, self.batch_size) + states = torch.FloatTensor([t[0] for t in minibatch]) + actions = torch.LongTensor([t[1] for t in minibatch]) + rewards = torch.FloatTensor([t[2] for t in minibatch]) + next_states = torch.FloatTensor([t[3] for t in minibatch]) + dones = torch.FloatTensor([t[4] for t in minibatch]) + + current_q_values = self.model(states).gather(1, actions.unsqueeze(1)) + next_q_values = self.model(next_states).max(1)[0].detach() + target_q_values = rewards + (1 - dones) * self.gamma * next_q_values + + loss = nn.MSELoss()(current_q_values.squeeze(), target_q_values) + + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + if self.epsilon > self.epsilon_min: + self.epsilon *= self.epsilon_decay + + def save(self, filename): + """Save model.""" + torch.save(self.model.state_dict(), filename) + + def load(self, filename): + """Load model.""" + self.model.load_state_dict(torch.load(filename)) + +# Usage +env = GridWorld() +state_size = env.size * env.size +action_size = 4 + +agent = DQNAgent(state_size, action_size) + +episodes = 500 +for episode in range(episodes): + state = env.reset() + state_flat = np.array(state).flatten() + done = False + total_reward = 0 + + while not done: + action = agent.act(state_flat) + next_state, reward, done = env.step(env.actions[action]) + next_state_flat = np.array(next_state).flatten() + + agent.remember(state_flat, action, reward, next_state_flat, done) + state = next_state + state_flat = next_state_flat + total_reward += reward + + agent.replay() + + if episode % 50 == 0: + print(f"Episode {episode}, Total Reward: {total_reward}, Epsilon: {agent.epsilon:.3f}") +``` + +### 3. Policy Gradient Implementation +Implement REINFORCE algorithm. + +```python +class PolicyGradientAgent: + def __init__(self, state_size, action_size, learning_rate=0.01): + self.state_size = state_size + self.action_size = action_size + self.gamma = 0.99 + + # Policy network + self.policy = nn.Sequential( + nn.Linear(state_size, 128), + nn.ReLU(), + nn.Linear(128, action_size), + nn.Softmax(dim=-1) + ) + + self.optimizer = optim.Adam(self.policy.parameters(), lr=learning_rate) + + def choose_action(self, state): + """Choose action based on policy.""" + state = torch.FloatTensor(state) + probs = self.policy(state) + + # Sample from probability distribution + action_dist = torch.distributions.Categorical(probs) + action = action_dist.sample() + + return action.item(), action_dist.log_prob(action) + + def update_policy(self, rewards, log_probs): + """Update policy using REINFORCE.""" + returns = [] + R = 0 + + # Calculate discounted returns + for r in reversed(rewards): + R = r + self.gamma * R + returns.insert(0, R) + + returns = torch.FloatTensor(returns) + log_probs = torch.stack(log_probs) + + # Calculate loss + policy_loss = [] + for log_prob, R in zip(log_probs, returns): + policy_loss.append(-log_prob * R) + + policy_loss = torch.stack(policy_loss).sum() + + # Update policy + self.optimizer.zero_grad() + policy_loss.backward() + self.optimizer.step() + +# Training +env = GridWorld() +agent = PolicyGradientAgent(state_size=env.size*env.size, action_size=4) + +episodes = 1000 +for episode in range(episodes): + state = env.reset() + state_flat = np.array(state).flatten() + done = False + rewards = [] + log_probs = [] + + while not done: + action, log_prob = agent.choose_action(state_flat) + next_state, reward, done = env.step(env.actions[action]) + + rewards.append(reward) + log_probs.append(log_prob) + + state = next_state + state_flat = np.array(state).flatten() + + agent.update_policy(rewards, log_probs) + + if episode % 100 == 0: + print(f"Episode {episode}, Total Reward: {sum(rewards)}") +``` + +## Constraints +- **Training Time**: RL requires many episodes to converge +- **Hyperparameter Sensitivity**: Performance highly depends on hyperparameter tuning +- **Sample Efficiency**: Traditional RL is sample inefficient +- **Exploration vs. Exploitation**: Balancing exploration and exploitation is crucial +- **Reward Design**: Poor reward design leads to unexpected behaviors +- **Computational Resources**: Deep RL requires significant computational power + +## Expected Output +Intelligent agents that learn optimal policies through experience, capable of making sequential decisions in complex environments. diff --git a/TRAE-Skills/ai_engineering/Speech_to_Text_Whisper.md b/TRAE-Skills/ai_engineering/Speech_to_Text_Whisper.md new file mode 100644 index 0000000000000000000000000000000000000000..1610049b4da0a7f2b421bfc351cf636bf2c51d59 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Speech_to_Text_Whisper.md @@ -0,0 +1,88 @@ +# Skill: Speech-to-Text Implementation (Whisper) + +## Purpose +To implement robust audio transcription and translation using OpenAI's Whisper model, handling large files, various formats, and specialized vocabulary. + +## When to Use +- When converting user voice commands to text in a web/mobile app. +- When transcribing long-form content (podcasts, meetings). +- When you need high-accuracy transcription for non-English languages. + +## Procedure + +### 1. Handling Large Files (Splitting with FFmpeg) +Whisper API has a 25MB limit. Use FFmpeg to split or compress. + +```bash +# Split audio into 10-minute segments +ffmpeg -i input.mp3 -f segment -segment_time 600 -c copy out%03d.mp3 + +# Compress to low-bitrate mono MP3 (saves space while keeping speech clear) +ffmpeg -i input.wav -ac 1 -ar 16000 -ab 32k output.mp3 +``` + +### 2. Basic Transcription (Node.js) +Integrate the OpenAI SDK for transcription. + +```typescript +import fs from 'fs'; +import OpenAI from 'openai'; + +const openai = new OpenAI(); + +async function transcribe(filePath: string) { + const response = await openai.audio.transcriptions.create({ + file: fs.createReadStream(filePath), + model: "whisper-1", + language: "en", // Optional but improves accuracy + response_format: "verbose_json", // Gives timestamps + prompt: "The transcript is about a software architecture meeting.", // Helps with context/acronyms + }); + + return response; +} +``` + +### 3. Handling Timestamps & Subtitles +Use the `srt` or `vtt` format for video captions. + +```typescript +const srtTranscription = await openai.audio.transcriptions.create({ + file: fs.createReadStream("video_audio.mp3"), + model: "whisper-1", + response_format: "srt", +}); + +fs.writeFileSync("subtitles.srt", srtTranscription); +``` + +### 4. Specialized Vocabulary (Prompts) +Pass technical terms or proper nouns in the `prompt` parameter to ensure correct spelling. + +```typescript +// Prompt example to ensure technical terms are spelled correctly +const prompt = "The speakers discuss Kubernetes, Istio, and gRPC."; +``` + +### 5. Local Execution (Python/Faster-Whisper) +For privacy or high volume, use `faster-whisper` locally. + +```python +from faster_whisper import WhisperModel + +model_size = "large-v3" +model = WhisperModel(model_size, device="cuda", compute_type="float16") + +segments, info = model.transcribe("audio.mp3", beam_size=5) + +for segment in segments: + print("[%.2fs -> %.2fs] %s" % (segment.start, segment.end, segment.text)) +``` + +## Constraints +- **File Size**: 25MB limit on API. Must split/compress manually. +- **Latency**: Not real-time (usually 5-15s for a 1-minute clip). +- **Privacy**: API usage sends data to OpenAI. Use local Whisper for sensitive data. + +## Expected Output +Highly accurate, time-stamped text representing the spoken audio, optionally translated into English. diff --git a/TRAE-Skills/ai_engineering/Stream_Responses.md b/TRAE-Skills/ai_engineering/Stream_Responses.md new file mode 100644 index 0000000000000000000000000000000000000000..33465b8fbc8ae9d8bae05af88b6fd06a989a80da --- /dev/null +++ b/TRAE-Skills/ai_engineering/Stream_Responses.md @@ -0,0 +1,321 @@ +# Skill: Stream Responses + +## Purpose +To deliver LLM responses in real-time chunks, improving user experience by providing immediate feedback and reducing perceived latency. + +## When to Use +- When building chat interfaces +- When processing long responses +- When users need immediate feedback +- When implementing real-time AI interactions + +## Procedure + +### 1. Basic Streaming Setup +Implement basic streaming with OpenAI API. + +```python +from openai import OpenAI +import sys + +client = OpenAI() + +def stream_response(prompt, model="gpt-4"): + """Stream a basic response from the LLM.""" + print("Assistant: ", end="", flush=True) + + stream = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + stream=True + ) + + for chunk in stream: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + print(content, end="", flush=True) + yield content + + print() # New line after completion + +# Usage +for chunk in stream_response("Explain quantum computing in simple terms"): + pass # Process chunks if needed +``` + +### 2. Streaming with Buffering +Implement buffering for more controlled output. + +```python +def buffered_stream(prompt, buffer_size=10, delay=0.1): + """Stream with buffering to prevent choppy output.""" + import time + + buffer = [] + stream = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + stream=True + ) + + for chunk in stream: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + buffer.append(content) + + if len(buffer) >= buffer_size: + print("".join(buffer), end="", flush=True) + buffer = [] + time.sleep(delay) + + # Print remaining buffer + if buffer: + print("".join(buffer), end="", flush=True) + print() + +# Usage +buffered_stream("Write a short story about a robot learning to paint") +``` + +### 3. Async Streaming +Implement asynchronous streaming for better performance. + +```python +import asyncio +from openai import AsyncOpenAI + +async_client = AsyncOpenAI() + +async def async_stream(prompt): + """Stream responses asynchronously.""" + stream = await async_client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + stream=True + ) + + full_response = "" + async for chunk in stream: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + print(content, end="", flush=True) + full_response += content + + print() + return full_response + +# Usage +async def main(): + response = await async_stream("What are the benefits of async programming?") + +asyncio.run(main()) +``` + +### 4. Multi-User Streaming +Handle streaming for multiple concurrent users. + +```python +class StreamingManager: + def __init__(self): + self.active_streams = {} + + async def stream_to_user(self, user_id, prompt): + """Stream response to a specific user.""" + self.active_streams[user_id] = True + + try: + stream = await async_client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + stream=True + ) + + async for chunk in stream: + if not self.active_streams.get(user_id, False): + print(f"Stream cancelled for user {user_id}") + break + + if chunk.choices[0].delta.content is not None: + # In real implementation, send to user via WebSocket + content = chunk.choices[0].delta.content + print(f"User {user_id}: {content}", end="", flush=True) + + print() + finally: + self.active_streams.pop(user_id, None) + + def cancel_stream(self, user_id): + """Cancel stream for a specific user.""" + self.active_streams[user_id] = False + +# Usage +manager = StreamingManager() + +async def handle_multiple_users(): + tasks = [ + manager.stream_to_user(1, "Tell me a joke"), + manager.stream_to_user(2, "Explain machine learning") + ] + await asyncio.gather(*tasks) + +asyncio.run(handle_multiple_users()) +``` + +### 5. Process Streaming Output +Process chunks during streaming. + +```python +class StreamProcessor: + def __init__(self): + self.collected_chunks = [] + self.processed_chunks = [] + + def process_chunk(self, chunk): + """Process individual chunks during streaming.""" + # Example: filter or transform chunks + processed = chunk.replace("**", "").replace("__", "") + self.processed_chunks.append(processed) + return processed + + def stream_with_processing(self, prompt): + """Stream and process chunks in real-time.""" + stream = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + stream=True + ) + + for chunk in stream: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + self.collected_chunks.append(content) + + processed = self.process_chunk(content) + print(processed, end="", flush=True) + + print() + return "".join(self.processed_chunks) + +# Usage +processor = StreamProcessor() +result = processor.stream_with_processing("Write markdown formatted text with bold and italics") +``` + +### 6. Web Integration with Streaming +Integrate streaming with web frameworks. + +```python +from fastapi import FastAPI +from fastapi.responses import StreamingResponse +import json + +app = FastAPI() + +def generate_stream(prompt): + """Generator for FastAPI streaming response.""" + stream = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + stream=True + ) + + for chunk in stream: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + # Format as SSE (Server-Sent Events) + data = json.dumps({"content": content}) + yield f"data: {data}\n\n" + + yield "data: [DONE]\n\n" + +@app.post("/chat") +async def chat_endpoint(prompt: str): + """Endpoint for streaming chat responses.""" + return StreamingResponse( + generate_stream(prompt), + media_type="text/event-stream" + ) + +# For client-side JavaScript: +""" +const response = await fetch('/chat', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({prompt: 'Hello'}) +}); + +const reader = response.body.getReader(); +const decoder = new TextDecoder(); + +while (true) { + const {done, value} = await reader.read(); + if (done) break; + + const chunk = decoder.decode(value); + // Process chunk and update UI +} +""" +``` + +### 7. Streaming with Metadata +Include metadata with streaming responses. + +```python +def stream_with_metadata(prompt): + """Stream response with additional metadata.""" + stream = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + stream=True + ) + + total_tokens = 0 + start_time = time.time() + + for i, chunk in enumerate(stream): + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + total_tokens += len(content.split()) + + metadata = { + "chunk": i, + "tokens": total_tokens, + "elapsed": time.time() - start_time + } + + yield { + "content": content, + "metadata": metadata + } + + # Final metadata + final_metadata = { + "total_chunks": i + 1, + "estimated_tokens": total_tokens, + "total_time": time.time() - start_time + } + + yield { + "content": "", + "metadata": final_metadata, + "complete": True + } + +# Usage +for response in stream_with_metadata("Write a 500-word essay on AI"): + if response.get("complete"): + print(f"\nCompleted in {response['metadata']['total_time']:.2f}s") + else: + print(response["content"], end="", flush=True) +``` + +## Constraints +- **Token Counting**: Streaming makes exact token counting difficult +- **Error Handling**: Handle connection failures mid-stream gracefully +- **Buffer Size**: Balance between real-time feedback and choppy output +- **Memory Usage**: Be careful with memory for very long responses +- **Cancellation**: Implement proper cancellation for user interruptions +- **Format Maintenance**: Streaming may break markdown formatting temporarily + +## Expected Output +Real-time streaming responses that provide immediate user feedback, with proper error handling and integration capabilities for various applications. diff --git a/TRAE-Skills/ai_engineering/Structured_Output_Parsing.md b/TRAE-Skills/ai_engineering/Structured_Output_Parsing.md new file mode 100644 index 0000000000000000000000000000000000000000..1ec29ee7aab2a7a0bbecb80723e3bd8373073101 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Structured_Output_Parsing.md @@ -0,0 +1,301 @@ +# Skill: Structured Output Parsing + +## Purpose +To force LLMs to output data in specific, parseable formats like JSON, enabling reliable integration with applications and automated workflows. + +## When to Use +- When building APIs that need structured responses +- When extracting data from unstructured text +- When generating code, configuration files, or data formats +- When requiring consistent output formatting for downstream processing + +## Procedure + +### 1. Basic JSON Output +Request JSON format in your prompt. + +```python +from openai import OpenAI +import json + +client = OpenAI() + +def get_json_response(prompt, schema_description): + """Get structured JSON response from LLM.""" + full_prompt = f""" + {prompt} + + Provide your response as a JSON object following this structure: + {schema_description} + + Return only the JSON, no additional text. + """ + + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": full_prompt}], + temperature=0 # Lower temperature for more consistent structure + ) + + # Parse JSON response + try: + return json.loads(response.choices[0].message.content) + except json.JSONDecodeError: + # Fallback: ask model to fix the JSON + return fix_json_parse(response.choices[0].message.content) + +# Example usage +person_info = get_json_response( + "Extract information from: 'John Doe is a 35-year-old software engineer from New York.'", + """ + { + "name": "string", + "age": "number", + "occupation": "string", + "location": "string" + } + """ +) +print(person_info) +``` + +### 2. Pydantic Models for Validation +Use Pydantic for robust schema validation. + +```python +from pydantic import BaseModel, Field +from typing import List, Optional + +class Person(BaseModel): + name: str = Field(description="Full name of the person") + age: int = Field(description="Age in years", ge=0, le=150) + occupation: str = Field(description="Job title or occupation") + location: Optional[str] = Field(default=None, description="City or location") + +class ProductReview(BaseModel): + product_name: str + rating: int = Field(ge=1, le=5) + review_text: str + pros: List[str] + cons: List[str] + would_recommend: bool + +def extract_structured_data(text, model_class): + """Extract structured data using Pydantic model.""" + schema = model_class.model_json_schema() + + prompt = f""" + Extract information from the following text and return it as JSON: + + Text: {text} + + Return JSON matching this schema: + {json.dumps(schema, indent=2)} + + Return only the JSON, no additional text. + """ + + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + temperature=0 + ) + + try: + data = json.loads(response.choices[0].message.content) + return model_class(**data) + except Exception as e: + print(f"Error parsing response: {e}") + return None + +# Example usage +review_text = """ +I bought the Acme Widget Pro last month. Overall I'd give it 4 stars. +The build quality is excellent and battery life is amazing. +However, the price is quite high and the app interface is confusing. +Despite these issues, I would recommend it to others. +""" + +review = extract_structured_data(review_text, ProductReview) +if review: + print(f"Product: {review.product_name}") + print(f"Rating: {review.rating}/5") + print(f"Pros: {', '.join(review.pros)}") +``` + +### 3. Few-Shot JSON Examples +Provide examples to improve JSON formatting. + +```python +def extract_with_few_shots(text): + """Extract structured data with few-shot examples.""" + prompt = """ + Example 1: + Input: "Sarah Connor, 28, works as a data analyst in Boston." + Output: {"name": "Sarah Connor", "age": 28, "occupation": "data analyst", "location": "Boston"} + + Example 2: + Input: "Mike Ross is a 32-year-old lawyer from New York." + Output: {"name": "Mike Ross", "age": 32, "occupation": "lawyer", "location": "New York"} + + Example 3: + Input: "Emily Chen, 25, graphic designer, San Francisco" + Output: {"name": "Emily Chen", "age": 25, "occupation": "graphic designer", "location": "San Francisco"} + + Now extract from this input: + Input: "{text}" + Output:""" + + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + temperature=0 + ) + + return json.loads(response.choices[0].message.content) + +# Usage +result = extract_with_few_shots("James Wilson, 42, architect, Chicago") +print(result) +``` + +### 4. Structured Output with Function Calling +Use function calling for guaranteed structured output. + +```python +def structured_output_with_functions(text): + """Use function calling for structured output.""" + function_def = { + "name": "extract_person_info", + "description": "Extract person information from text", + "parameters": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + "occupation": {"type": "string"}, + "location": {"type": "string"} + }, + "required": ["name", "age", "occupation"] + } + } + + response = client.chat.completions.create( + model="gpt-4", + messages=[{ + "role": "user", + "content": f"Extract person information from: {text}" + }], + functions=[function_def], + function_call={"name": "extract_person_info"} + ) + + function_call = response.choices[0].message.function_call + return json.loads(function_call.arguments) + +# Example +person = structured_output_with_functions( + "Dr. Lisa Anderson, 38, cardiologist at Mayo Clinic, Rochester" +) +print(person) +``` + +### 5. Complex Nested Structures +Handle complex nested JSON structures. + +```python +class Company(BaseModel): + name: str + founded_year: int + headquarters: str + employees: List[str] + departments: dict + +def extract_company_info(text): + """Extract complex nested company information.""" + prompt = f""" + Extract company information from the following text and return it as JSON. + Include all employees mentioned and their departments. + + Text: {text} + + Return JSON matching this structure: + {json.dumps(Company.model_json_schema(), indent=2)} + + Return only valid JSON. + """ + + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + temperature=0 + ) + + try: + data = json.loads(response.choices[0].message.content) + return Company(**data) + except Exception as e: + print(f"Error: {e}") + return None + +# Example +company_text = """ +TechCorp Inc. was founded in 2010 and is headquartered in Austin, Texas. +The company has three main departments: Engineering, Sales, and Marketing. +Key employees include: +- John Smith (CEO, Engineering department) +- Sarah Johnson (VP of Sales) +- Michael Chen (CTO, Engineering) +- Emily Davis (Marketing Director) +""" + +company = extract_company_info(company_text) +if company: + print(f"Company: {company.name}") + print(f"Departments: {company.departments.keys()}") +``` + +### 6. Error Recovery and Retry Logic +Implement robust error handling for JSON parsing. + +```python +def extract_with_retry(text, model_class, max_retries=3): + """Extract structured data with retry logic.""" + schema = model_class.model_json_schema() + + for attempt in range(max_retries): + prompt = f""" + Extract information as JSON following this schema: + {json.dumps(schema, indent=2)} + + Text: {text} + + {'Only return the JSON object, no other text.' if attempt == 0 else 'Your previous response was not valid JSON. Please ensure you return ONLY a valid JSON object, nothing else.'} + """ + + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + temperature=0 + ) + + try: + data = json.loads(response.choices[0].message.content) + return model_class(**data) + except Exception as e: + if attempt == max_retries - 1: + print(f"Failed after {max_retries} attempts: {e}") + return None + print(f"Attempt {attempt + 1} failed, retrying...") +``` + +## Constraints +- **Model Consistency**: Not all models follow instructions perfectly - use function calling when possible +- **Temperature**: Use low temperature (0-0.3) for more consistent formatting +- **Complexity**: Very complex structures may require multiple extraction steps +- **Validation**: Always validate parsed data before using it +- **Error Handling**: Implement robust error handling for malformed responses +- **Token Limits**: Large schemas consume tokens - keep them as minimal as possible + +## Expected Output +Reliable structured data extraction from unstructured text, with properly validated and typed results that can be directly used in applications. diff --git a/TRAE-Skills/ai_engineering/Time_Series_Forecasting.md b/TRAE-Skills/ai_engineering/Time_Series_Forecasting.md new file mode 100644 index 0000000000000000000000000000000000000000..8ecb23659a7416f09186cbff646986dba4f3757e --- /dev/null +++ b/TRAE-Skills/ai_engineering/Time_Series_Forecasting.md @@ -0,0 +1,94 @@ +# Skill: Time Series Forecasting + +## Purpose +To predict future values based on previously observed time-ordered data, considering trends, seasonality, and exogenous variables. + +## When to Use +- When forecasting sales, inventory, stock prices, or resource demand +- When identifying anomalies in system metrics over time +- When working with IoT sensor data + +## Procedure + +### 1. Identify the Time Series Characteristics +Analyze the data for: +- **Trend**: Long-term upward or downward movement. +- **Seasonality**: Repeating patterns at fixed intervals (daily, weekly, yearly). +- **Stationarity**: Whether statistical properties (mean, variance) change over time. + +### 2. Choose the Forecasting Model +Select an appropriate algorithm based on data complexity: +- **ARIMA / SARIMA**: Traditional statistical models. Best for univariate data with clear seasonality. +- **Prophet (by Meta)**: Excellent for business time series with daily observations and strong seasonal effects. +- **XGBoost / LightGBM**: Effective for tabular time series with many exogenous features. +- **LSTMs / Transformers (e.g., Temporal Fusion Transformer)**: Best for complex, non-linear relationships and long sequences. + +### 3. Prophet Implementation Example + +**Setup and Fitting**: +```python +from prophet import Prophet +import pandas as pd + +# Data must have 'ds' (datestamp) and 'y' (target) columns +df = pd.read_csv('sales_data.csv') +df.rename(columns={'date': 'ds', 'sales': 'y'}, inplace=True) + +# Initialize Prophet model +m = Prophet( + yearly_seasonality=True, + weekly_seasonality=True, + daily_seasonality=False +) + +# Add custom holidays or exogenous variables if necessary +m.add_country_holidays(country_name='US') + +# Fit the model +m.fit(df) +``` + +**Forecasting**: +```python +# Create future dates +future = m.make_future_dataframe(periods=365) + +# Predict future values +forecast = m.predict(future) + +# Plotting results +fig1 = m.plot(forecast) +fig2 = m.plot_components(forecast) +``` + +### 4. Evaluation Metrics +Use appropriate error metrics for time series: +- **MAE (Mean Absolute Error)**: Easy to interpret, robust to outliers. +- **RMSE (Root Mean Squared Error)**: Penalizes large errors heavily. +- **MAPE (Mean Absolute Percentage Error)**: Useful for comparing relative performance across different scales. + +```python +from sklearn.metrics import mean_absolute_error, mean_squared_error +import numpy as np + +mae = mean_absolute_error(y_true, y_pred) +rmse = np.sqrt(mean_squared_error(y_true, y_pred)) +``` + +### 5. Cross-Validation +Do not use standard K-Fold cross-validation. Use Time Series Split to respect temporal order. + +```python +from sklearn.model_selection import TimeSeriesSplit + +tscv = TimeSeriesSplit(n_splits=5) +for train_index, test_index in tscv.split(X): + X_train, X_test = X[train_index], X[test_index] + y_train, y_test = y[train_index], y[test_index] + # Train and evaluate model +``` + +## Best Practices +- Always plot your data before modeling. Visual inspection reveals obvious anomalies or structural breaks. +- Use baseline models (like naive forecasting or moving average) to establish a performance floor before moving to complex models. +- Handle missing values carefully; avoid interpolating over large gaps without justification. \ No newline at end of file diff --git a/TRAE-Skills/ai_engineering/Time_Series_Forecasting_LSTM.md b/TRAE-Skills/ai_engineering/Time_Series_Forecasting_LSTM.md new file mode 100644 index 0000000000000000000000000000000000000000..6325ac4b8686d171d39195af06e7422a00729ad2 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Time_Series_Forecasting_LSTM.md @@ -0,0 +1,233 @@ +# Skill: Time Series Forecasting with LSTM Networks + +## Purpose +To build accurate time series prediction models using Long Short-Term Memory (LSTM) neural networks. + +## When to Use +- For stock price prediction +- When forecasting sales or demand +- For predicting energy consumption +- When analyzing financial markets +- For weather forecasting with sequential data + +## Procedure + +### 1. Data Preparation +Prepare time series data for LSTM. + +```python +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.preprocessing import MinMaxScaler +from sklearn.model_selection import train_test_split + +# Load sample data +df = pd.read_csv('time_series_data.csv', parse_dates=['date'], index_col='date') +data = df[['value']].values + +# Normalize data +scaler = MinMaxScaler(feature_range=(0, 1)) +scaled_data = scaler.fit_transform(data) + +# Create sequences +def create_sequences(data, time_steps=60): + X, y = [], [] + for i in range(time_steps, len(data)): + X.append(data[i-time_steps:i, 0]) + y.append(data[i, 0]) + return np.array(X), np.array(y) + +time_steps = 60 +X, y = create_sequences(scaled_data, time_steps) + +# Reshape for LSTM (samples, time steps, features) +X = np.reshape(X, (X.shape[0], X.shape[1], 1)) + +# Split into train and test +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=False) +``` + +### 2. Build LSTM Model +Create an LSTM model with Keras. + +```python +from tensorflow.keras.models import Sequential +from tensorflow.keras.layers import LSTM, Dense, Dropout +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint + +model = Sequential([ + LSTM(50, return_sequences=True, input_shape=(X_train.shape[1], 1)), + Dropout(0.2), + LSTM(50, return_sequences=False), + Dropout(0.2), + Dense(25), + Dense(1) +]) + +model.compile(optimizer=Adam(learning_rate=0.001), loss='mean_squared_error') + +model.summary() +``` + +### 3. Train the Model +Train the LSTM model. + +```python +callbacks = [ + EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True), + ModelCheckpoint('best_model.h5', monitor='val_loss', save_best_only=True) +] + +history = model.fit( + X_train, y_train, + batch_size=32, + epochs=100, + validation_data=(X_test, y_test), + callbacks=callbacks +) + +# Plot training history +plt.figure(figsize=(12, 6)) +plt.plot(history.history['loss'], label='Training Loss') +plt.plot(history.history['val_loss'], label='Validation Loss') +plt.title('Model Loss') +plt.xlabel('Epoch') +plt.ylabel('Loss') +plt.legend() +plt.show() +``` + +### 4. Make Predictions +Generate predictions with the trained model. + +```python +# Predict on test data +predictions = model.predict(X_test) +predictions = scaler.inverse_transform(predictions) +y_test_actual = scaler.inverse_transform(y_test.reshape(-1, 1)) + +# Plot results +train = df[:-len(y_test)] +valid = df[-len(y_test):] +valid['Predictions'] = predictions + +plt.figure(figsize=(16, 8)) +plt.title('Time Series Prediction') +plt.xlabel('Date', fontsize=18) +plt.ylabel('Value', fontsize=18) +plt.plot(train['value']) +plt.plot(valid[['value', 'Predictions']]) +plt.legend(['Training Data', 'Actual Value', 'Predicted Value'], loc='lower right') +plt.show() +``` + +### 5. Evaluate the Model +Calculate evaluation metrics. + +```python +from sklearn.metrics import mean_squared_error, mean_absolute_error, mean_absolute_percentage_error + +rmse = np.sqrt(mean_squared_error(y_test_actual, predictions)) +mae = mean_absolute_error(y_test_actual, predictions) +mape = mean_absolute_percentage_error(y_test_actual, predictions) + +print(f'RMSE: {rmse:.2f}') +print(f'MAE: {mae:.2f}') +print(f'MAPE: {mape:.2%}') +``` + +### 6. Multi-Step Forecasting +Predict multiple future time steps. + +```python +def multi_step_forecast(model, last_sequence, steps, scaler, time_steps): + forecast = [] + current_sequence = last_sequence.copy() + + for _ in range(steps): + # Predict next step + prediction = model.predict(current_sequence.reshape(1, time_steps, 1), verbose=0) + + # Store prediction + forecast.append(prediction[0, 0]) + + # Update sequence + current_sequence = np.roll(current_sequence, -1) + current_sequence[-1] = prediction[0, 0] + + # Inverse transform + forecast = scaler.inverse_transform(np.array(forecast).reshape(-1, 1)) + return forecast.flatten() + +# Get the last sequence from training data +last_sequence = X_test[-1] + +# Forecast next 30 days +forecast_steps = 30 +forecast = multi_step_forecast(model, last_sequence, forecast_steps, scaler, time_steps) + +# Create future dates +last_date = df.index[-1] +future_dates = pd.date_range(start=last_date + pd.Timedelta(days=1), periods=forecast_steps) + +# Plot forecast +plt.figure(figsize=(16, 8)) +plt.plot(df['value'], label='Historical Data') +plt.plot(future_dates, forecast, label='Forecast', linestyle='--') +plt.title('Multi-Step Time Series Forecast') +plt.xlabel('Date') +plt.ylabel('Value') +plt.legend() +plt.show() +``` + +### 7. Multivariate LSTM +Use multiple features for prediction. + +```python +# Load data with multiple features +df = pd.read_csv('multivariate_data.csv', parse_dates=['date'], index_col='date') +features = ['value', 'feature1', 'feature2', 'feature3'] +data = df[features].values + +# Normalize all features +scaler = MinMaxScaler(feature_range=(0, 1)) +scaled_data = scaler.fit_transform(data) + +# Create sequences with multiple features +def create_multivariate_sequences(data, time_steps=60): + X, y = [], [] + for i in range(time_steps, len(data)): + X.append(data[i-time_steps:i, :]) # All features + y.append(data[i, 0]) # Target is first feature + return np.array(X), np.array(y) + +time_steps = 60 +X, y = create_multivariate_sequences(scaled_data, time_steps) + +# Build multivariate LSTM model +model = Sequential([ + LSTM(100, return_sequences=True, input_shape=(X.shape[1], X.shape[2])), + Dropout(0.3), + LSTM(100, return_sequences=False), + Dropout(0.3), + Dense(50), + Dense(1) +]) + +model.compile(optimizer=Adam(learning_rate=0.001), loss='mean_squared_error') +``` + +## Best Practices +- **Normalize/Standardize**: Always normalize time series data +- **Sequence Length**: Choose appropriate time step window +- **Validation**: Use walk-forward validation for time series +- **Regularization**: Use dropout to prevent overfitting +- **Early Stopping**: Stop training when validation loss stops improving +- **Feature Engineering**: Create meaningful features (lags, rolling stats) +- **Ensemble**: Combine with ARIMA, Prophet for better results +- **Hyperparameter Tuning**: Optimize layers, units, learning rate +- **Monitor**: Track performance on both train and validation sets +- **Update**: Retrain model periodically with new data diff --git a/TRAE-Skills/ai_engineering/Token_Optimization.md b/TRAE-Skills/ai_engineering/Token_Optimization.md new file mode 100644 index 0000000000000000000000000000000000000000..fc6c6531f3459403f9caa9105b813a1a33a4dd8e --- /dev/null +++ b/TRAE-Skills/ai_engineering/Token_Optimization.md @@ -0,0 +1,245 @@ +# Skill: Token Optimization + +## Purpose +To minimize token usage while maintaining output quality, reducing API costs and improving response times. + +## When to Use +- When working with large documents or contexts +- When building cost-sensitive applications +- When processing multiple documents +- When optimizing for faster responses + +## Procedure + +### 1. Prune Redundant Content +Remove unnecessary information from prompts. + +```python +def prune_redundant_content(text): + """Remove redundant phrases and content.""" + # Common redundant phrases to remove + redundant_phrases = [ + "please note that", + "it is important to mention", + "as previously stated", + "in conclusion", + "additionally", + "furthermore", + "moreover" + ] + + pruned = text + for phrase in redundant_phrases: + pruned = pruned.replace(phrase, "") + + # Remove multiple spaces + pruned = " ".join(pruned.split()) + + return pruned + +# Example +original = """Please note that it is important to mention that the document +contains multiple sections. Furthermore, additionally, it has various topics.""" + +optimized = prune_redundant_content(original) +print(f"Original: {len(original)} tokens") +print(f"Optimized: {len(optimized)} tokens") +``` + +### 2. Use Efficient Prompt Structures +Structure prompts to maximize information density. + +```python +# INEFFICIENT - verbose prompt +inefficient_prompt = """ +I would like you to please help me by analyzing the following text. +Please provide me with a summary of the main points. +The text is as follows: {text} +""" + +# EFFICIENT - concise prompt +efficient_prompt = """Summarize the main points: {text}""" + +# Even more efficient with examples +few_shot_efficient = """Text: {text1} +Summary: {summary1} + +Text: {text2} +Summary: {summary2} + +Text: {text} +Summary:""" +``` + +### 3. Chunk and Summarize Large Documents +Process large documents in chunks with progressive summarization. + +```python +def chunk_text(text, max_chunk_size=2000): + """Split text into chunks of roughly equal token count.""" + words = text.split() + chunks = [] + current_chunk = [] + current_length = 0 + + for word in words: + current_chunk.append(word) + current_length += 1 + + if current_length >= max_chunk_size: + chunks.append(" ".join(current_chunk)) + current_chunk = [] + current_length = 0 + + if current_chunk: + chunks.append(" ".join(current_chunk)) + + return chunks + +def progressive_summarize(text): + """Summarize large document progressively.""" + # Split into chunks + chunks = chunk_text(text, max_chunk_size=2000) + + # Summarize each chunk + chunk_summaries = [] + for chunk in chunks: + summary = client.chat.completions.create( + model="gpt-4", + messages=[{ + "role": "user", + "content": f"Summarize in 1-2 sentences: {chunk}" + }] + ) + chunk_summaries.append(summary.choices[0].message.content) + + # Combine chunk summaries + combined = " ".join(chunk_summaries) + + # Final summary + final_summary = client.chat.completions.create( + model="gpt-4", + messages=[{ + "role": "user", + "content": f"Create a cohesive summary: {combined}" + }] + ) + + return final_summary.choices[0].message.content +``` + +### 4. Use System Messages Effectively +Move static context to system messages. + +```python +# INEFFICIENT - repeating instructions in every message +messages_inefficient = [ + { + "role": "user", + "content": "You are a helpful assistant that provides concise answers. + Please answer this question: What is machine learning?" + } +] + +# EFFICIENT - use system message +messages_efficient = [ + { + "role": "system", + "content": "You are a helpful assistant. Provide concise answers in 2-3 sentences." + }, + { + "role": "user", + "content": "What is machine learning?" + } +] +``` + +### 5. Use Smaller Models When Appropriate +Choose the right model for the task. + +```python +def choose_model(task_complexity, token_count): + """Choose the most cost-effective model.""" + if token_count < 500 and task_complexity == "simple": + return "gpt-3.5-turbo" + elif token_count < 2000 and task_complexity in ["simple", "medium"]: + return "gpt-3.5-turbo" + else: + return "gpt-4" + +# Example usage +text = "Your text here..." +token_count = len(text.split()) * 1.3 # Rough estimate +model = choose_model("medium", token_count) + +response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": text}] +) +``` + +### 6. Implement Token Counting and Budgeting +Track and limit token usage. + +```python +import tiktoken + +def count_tokens(text, model="gpt-4"): + """Count tokens in text.""" + encoding = tiktoken.encoding_for_model(model) + return len(encoding.encode(text)) + +class TokenBudget: + def __init__(self, max_tokens=100000): + self.max_tokens = max_tokens + self.used_tokens = 0 + self.encoding = tiktoken.encoding_for_model("gpt-4") + + def can_process(self, text): + """Check if we can process this text within budget.""" + tokens = count_tokens(text) + return (self.used_tokens + tokens) <= self.max_tokens + + def process(self, text, function): + """Process text if within budget.""" + if not self.can_process(text): + raise Exception("Token budget exceeded!") + + tokens = count_tokens(text) + self.used_tokens += tokens + + return function(text) + + def get_usage(self): + """Get current token usage.""" + return { + "used": self.used_tokens, + "remaining": self.max_tokens - self.used_tokens, + "percentage": (self.used_tokens / self.max_tokens) * 100 + } + +# Usage +budget = TokenBudget(max_tokens=50000) + +def make_llm_call(text): + return client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": text}] + ) + +try: + result = budget.process("Your text here", make_llm_call) + print(budget.get_usage()) +except Exception as e: + print(f"Error: {e}") +``` + +## Constraints +- **Quality vs. Cost**: Aggressive optimization may impact output quality +- **Context Loss**: Removing too much content may lose important information +- **Model Limitations**: Different models have different capabilities +- **Token Estimation**: Token counts are estimates, not exact +- **Budget Planning**: Always include buffer for unexpected token usage + +## Expected Output +Reduced API costs (30-70% savings) while maintaining acceptable output quality through intelligent token optimization strategies. diff --git a/TRAE-Skills/ai_engineering/Vector_Database_Setup.md b/TRAE-Skills/ai_engineering/Vector_Database_Setup.md new file mode 100644 index 0000000000000000000000000000000000000000..8991cc548feedbddbe7e1ef3692a52ffc23dcba5 --- /dev/null +++ b/TRAE-Skills/ai_engineering/Vector_Database_Setup.md @@ -0,0 +1,97 @@ +# Skill: Vector Database Setup + +## Purpose +To provision and configure a vector database (Vector DB) for storing high-dimensional embeddings, enabling semantic search and RAG applications. + +## When to Use +- Implementing RAG (Retrieval-Augmented Generation). +- Building recommendation systems based on similarity. +- Implementing semantic search (search by meaning, not just keywords). + +## Procedure + +### 1. Choice of Database (Selection) +- **Pinecone**: Best for managed, serverless, and fast scaling. +- **pgvector**: Best for existing PostgreSQL users who want to keep data in one place. +- **Chroma**: Best for local development and simple prototyping. + +### 2. Implementation: Pinecone (Managed) +Install the client and initialize the index. + +```bash +npm install @pinecone-database/pinecone +``` + +```typescript +import { Pinecone } from '@pinecone-database/pinecone'; + +const pc = new Pinecone({ apiKey: process.env.PINECONE_API_KEY! }); + +async function setupIndex() { + await pc.createIndex({ + name: 'my-index', + dimension: 1536, // Must match embedding model (e.g., text-embedding-3-small) + metric: 'cosine', + spec: { + serverless: { + cloud: 'aws', + region: 'us-east-1' + } + } + }); +} +``` + +### 3. Implementation: pgvector (PostgreSQL) +Enable the extension and create a table with a vector column. + +```sql +-- 1. Enable extension +CREATE EXTENSION IF NOT EXISTS vector; + +-- 2. Create table +CREATE TABLE documents ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + content text, + metadata jsonb, + embedding vector(1536) -- Match your model's dimensions +); + +-- 3. Create index for fast search (HNSW is recommended) +CREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops); +``` + +### 4. Data Insertion (Upsert) +Always batch your insertions for efficiency. + +```typescript +// Pinecone example +const index = pc.index('my-index'); + +await index.upsert([ + { + id: 'doc1', + values: [0.1, 0.2, ...], // The embedding vector + metadata: { text: 'The actual content...', category: 'legal' } + } +]); +``` + +### 5. Querying (Semantic Search) +Perform a similarity search using a query vector. + +```typescript +const queryResponse = await index.query({ + vector: [0.1, 0.2, ...], // Vector of the user query + topK: 5, + includeMetadata: true, +}); +``` + +## Constraints +- **Dimension Matching**: The dimension of the index MUST exactly match the output dimension of your embedding model. +- **Metric Selection**: Use `cosine` for text; `euclidean` or `dotproduct` for other specific use cases. +- **Batch Limits**: Most vector DBs have limits on payload size per upsert (e.g., 2MB or 100 vectors). + +## Expected Output +A fully configured vector index ready for high-speed similarity searches and data retrieval. diff --git a/TRAE-Skills/ai_engineering/Vector_Databases_Pinecone_Weaviate.md b/TRAE-Skills/ai_engineering/Vector_Databases_Pinecone_Weaviate.md new file mode 100644 index 0000000000000000000000000000000000000000..be024fbd66b6a4a4e19928e2798be254ef32966f --- /dev/null +++ b/TRAE-Skills/ai_engineering/Vector_Databases_Pinecone_Weaviate.md @@ -0,0 +1,220 @@ +# Skill: Advanced Vector Databases (Pinecone & Weaviate) + +## Purpose +To build production-ready RAG, semantic search, and similarity systems with modern vector databases. + +## When to Use +- For building semantic search over large document collections +- When implementing RAG (Retrieval-Augmented Generation) +- For recommendation systems using embeddings +- When building question-answering systems +- For similarity search (images, text, audio) + +## Procedure + +### 1. Pinecone Setup & Indexing +Get started with Pinecone. + +```python +import os +from pinecone import Pinecone, ServerlessSpec +from langchain_openai import OpenAIEmbeddings +from langchain_core.documents import Document +from langchain_pinecone import PineconeVectorStore + +# Initialize Pinecone +pc = Pinecone(api_key=os.getenv("PINECONE_API_KEY")) + +# Create index +index_name = "rag-index" +if index_name not in pc.list_indexes().names(): + pc.create_index( + name=index_name, + dimension=1536, # text-embedding-3-small + metric="cosine", + spec=ServerlessSpec(cloud="aws", region="us-east-1") + ) + +# Connect to index +index = pc.Index(index_name) + +# Create embeddings +embeddings = OpenAIEmbeddings(model="text-embedding-3-small") + +# Sample documents +documents = [ + Document(page_content="LangGraph is a library for building stateful agents.", metadata={"source": "doc1", "category": "ai"}), + Document(page_content="Vector databases store embeddings for similarity search.", metadata={"source": "doc2", "category": "database"}), + Document(page_content="RAG combines retrieval with LLM generation.", metadata={"source": "doc3", "category": "ai"}), + Document(page_content="Pinecone is a serverless vector database.", metadata={"source": "doc4", "category": "database"}) +] + +# Add to vector store +vector_store = PineconeVectorStore.from_documents( + documents=documents, + embedding=embeddings, + index_name=index_name, + namespace="production" +) +``` + +### 2. Advanced Retrieval with Pinecone +Implement hybrid search and filtering. + +```python +# Semantic search +results = vector_store.similarity_search( + "What is LangGraph?", + k=3, + filter={"category": "ai"} # Metadata filtering +) + +# Similarity search with score +results_with_scores = vector_store.similarity_search_with_score( + "Tell me about vector databases", + k=3 +) + +# Hybrid search (dense + sparse) +# Use Pinecone's hybrid search with BM25 +from langchain_community.retrievers import PineconeHybridSearchRetriever +from pinecone_text.sparse import BM25Encoder + +bm25_encoder = BM25Encoder() +bm25_encoder.fit([d.page_content for d in documents]) + +hybrid_retriever = PineconeHybridSearchRetriever( + embeddings=embeddings, + sparse_encoder=bm25_encoder, + index=index, + namespace="production", + top_k=3 +) + +hybrid_results = hybrid_retriever.invoke("What is RAG?") +``` + +### 3. Weaviate Setup +Use Weaviate for self-hosted or managed vector search. + +```python +import weaviate +import weaviate.classes as wvc +from weaviate.classes.config import Property, DataType, Configure + +# Connect to Weaviate +client = weaviate.connect_to_wcs( + cluster_url=os.getenv("WEAVIATE_CLUSTER_URL"), + auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY")), + headers={ + "X-OpenAI-Api-Key": os.getenv("OPENAI_API_KEY") + } +) + +# Create collection +if not client.collections.exists("Document"): + collection = client.collections.create( + name="Document", + vectorizer_config=Configure.Vectorizer.text2vec_openai(model="text-embedding-3-small"), + generative_config=Configure.Generative.openai(model="gpt-4o"), + properties=[ + Property(name="content", data_type=DataType.TEXT), + Property(name="source", data_type=DataType.TEXT), + Property(name="category", data_type=DataType.TEXT), + Property(name="created_at", data_type=DataType.DATE) + ] + ) +else: + collection = client.collections.get("Document") + +# Add objects +with collection.batch.dynamic() as batch: + for doc in documents: + batch.add_object( + properties={ + "content": doc.page_content, + "source": doc.metadata["source"], + "category": doc.metadata["category"], + "created_at": "2024-01-01T00:00:00Z" + } + ) +``` + +### 4. Weaviate Generative Search (RAG) +Built-in generative search with Weaviate. + +```python +# Basic search +response = collection.query.near_text( + query="What is LangGraph?", + limit=3, + filters=wvc.query.Filter.by_property("category").equal("ai"), + return_properties=["content", "source"] +) + +# Generative search (RAG) +generative_response = collection.generate.near_text( + query="Explain LangGraph in simple terms", + limit=3, + single_prompt="Summarize this content: {content}" +) + +for obj in generative_response.objects: + print(f"Source: {obj.properties['source']}") + print(f"Generated: {obj.generated}") + print("---") + +# Grouped task +grouped_response = collection.generate.near_text( + query="Tell me about vector databases", + limit=5, + grouped_task="Write a comprehensive summary of all these documents" +) + +print("Grouped Summary:", grouped_response.generated) +``` + +### 5. Production Best Practices +Optimize vector DB for production. + +```python +# Pinecone optimization +# 1. Use namespaces for isolation +vector_store = PineconeVectorStore( + index=index, + embedding=embeddings, + namespace="tenant-123" +) + +# 2. Batch operations +from langchain.text_splitter import RecursiveCharacterTextSplitter + +text_splitter = RecursiveCharacterTextSplitter( + chunk_size=1000, + chunk_overlap=200 +) + +split_docs = text_splitter.split_documents(documents) + +# Add in batches +for i in range(0, len(split_docs), 100): + batch = split_docs[i:i+100] + vector_store.add_documents(batch) + +# 3. Weaviate: Use tenant isolation +multi_tenancy_config = Configure.multi_tenancy(enabled=True) + +# 4. Monitor usage +stats = index.describe_index_stats() +print(f"Total vectors: {stats['total_vector_count']}") +``` + +## Best Practices +- **Embedding Model**: Choose appropriate embedding model (dimensions, cost) +- **Chunking**: Use good text splitting strategy (chunk size, overlap) +- **Metadata**: Add rich metadata for filtering +- **Indexing**: Use batch operations for large datasets +- **Hybrid Search**: Combine vector + keyword search for better results +- **Namespaces/Tenants**: Isolate data for multi-tenant apps +- **Monitoring**: Monitor query latency and cost +- **Caching**: Cache frequent queries to reduce cost diff --git a/TRAE-Skills/architecture/API_Gateway_Pattern.md b/TRAE-Skills/architecture/API_Gateway_Pattern.md new file mode 100644 index 0000000000000000000000000000000000000000..dc465a370e02d7e3ad41ab933172719eef1c80fc --- /dev/null +++ b/TRAE-Skills/architecture/API_Gateway_Pattern.md @@ -0,0 +1,488 @@ +# Skill: API Gateway Pattern + +## Purpose +To provide a single entry point for microservices by implementing a centralized API gateway that handles routing, authentication, rate limiting, and cross-cutting concerns. + +## When to Use +- When building microservices architectures +- When you need a single entry point for multiple services +- When implementing cross-cutting concerns like auth, logging, rate limiting +- When aggregating responses from multiple services + +## Procedure + +### 1. Basic API Gateway Setup +Implement a basic API gateway with routing. + +```python +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse +from httpx import AsyncClient +import logging +from typing import Dict, Any +import asyncio + +class APIGateway: + """API Gateway for microservices.""" + + def __init__(self): + self.app = FastAPI(title="API Gateway") + self.services: Dict[str, str] = {} + self.client = AsyncClient() + self.logger = logging.getLogger("APIGateway") + + # Setup middleware + self._setup_middleware() + + # Setup routes + self._setup_routes() + + def _setup_middleware(self): + """Setup gateway middleware.""" + @self.app.middleware("http") + async def log_requests(request: Request, call_next): + start_time = asyncio.get_event_loop().time() + + # Log request + self.logger.info(f"Incoming request: {request.method} {request.url.path}") + + # Process request + response = await call_next(request) + + # Log response + process_time = asyncio.get_event_loop().time() - start_time + self.logger.info(f"Request completed in {process_time:.3f}s") + + response.headers["X-Process-Time"] = str(process_time) + return response + + def _setup_routes(self): + """Setup gateway routes.""" + + @self.app.get("/health") + async def health_check(): + return {"status": "healthy", "gateway": "API Gateway v1.0"} + + @self.app.api_route("/{service}/{path:path}", methods=["GET", "POST", "PUT", "DELETE"]) + async def proxy_request(service: str, path: str, request: Request): + """Proxy request to appropriate service.""" + if service not in self.services: + raise HTTPException(status_code=404, detail=f"Service '{service}' not found") + + service_url = self.services[service] + target_url = f"{service_url}/{path}" + + # Forward request + body = await request.body() + headers = dict(request.headers) + + try: + response = await self.client.request( + method=request.method, + url=target_url, + headers=headers, + content=body, + timeout=30.0 + ) + + return JSONResponse( + content=response.json(), + status_code=response.status_code, + headers=dict(response.headers) + ) + + except Exception as e: + self.logger.error(f"Error proxying request: {str(e)}") + raise HTTPException(status_code=503, detail="Service unavailable") + + def register_service(self, name: str, url: str): + """Register a microservice.""" + self.services[name] = url + self.logger.info(f"Registered service: {name} -> {url}") + + def get_app(self): + """Get FastAPI application.""" + return self.app + +# Usage +gateway = APIGateway() + +# Register services +gateway.register_service("users", "http://localhost:8001") +gateway.register_service("products", "http://localhost:8002") +gateway.register_service("orders", "http://localhost:8003") + +app = gateway.get_app() +``` + +### 2. Authentication and Authorization +Implement authentication middleware. + +```python +from fastapi import Security, HTTPException, status +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +import jwt +from datetime import datetime, timedelta + +security = HTTPBearer() + +class AuthMiddleware: + """Authentication middleware for API Gateway.""" + + def __init__(self, secret_key: str, algorithm: str = "HS256"): + self.secret_key = secret_key + self.algorithm = algorithm + + def create_token(self, user_id: str, permissions: list, expires_delta: timedelta = None): + """Create JWT token.""" + if expires_delta: + expire = datetime.utcnow() + expires_delta + else: + expire = datetime.utcnow() + timedelta(hours=1) + + payload = { + "user_id": user_id, + "permissions": permissions, + "exp": expire + } + + return jwt.encode(payload, self.secret_key, algorithm=self.algorithm) + + def verify_token(self, token: str) -> Dict[str, Any]: + """Verify JWT token.""" + try: + payload = jwt.decode(token, self.secret_key, algorithms=[self.algorithm]) + return payload + except jwt.ExpiredSignatureError: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Token expired" + ) + except jwt.JWTError: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token" + ) + + def check_permission(self, required_permission: str): + """Check if user has required permission.""" + async def permission_checker(credentials: HTTPAuthorizationCredentials = Security(security)): + payload = self.verify_token(credentials.credentials) + permissions = payload.get("permissions", []) + + if required_permission not in permissions: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Permission '{required_permission}' required" + ) + + return payload + + return permission_checker + +# Integrate with gateway +auth_middleware = AuthMiddleware(secret_key="your-secret-key") + +@gateway.app.get("/protected") +async def protected_endpoint(user_data: Dict = Depends(auth_middleware.check_permission("read:protected"))): + return {"message": "Access granted", "user": user_data["user_id"]} +``` + +### 3. Rate Limiting +Implement rate limiting for API protection. + +```python +from fastapi import Request, HTTPException +from collections import defaultdict +import time +from typing import Dict + +class RateLimiter: + """Rate limiter using sliding window algorithm.""" + + def __init__(self, requests_per_minute: int = 60): + self.requests_per_minute = requests_per_minute + self.requests: Dict[str, list] = defaultdict(list) + + def is_allowed(self, key: str) -> bool: + """Check if request is allowed.""" + now = time.time() + minute_ago = now - 60 + + # Remove old requests + self.requests[key] = [ + req_time for req_time in self.requests[key] + if req_time > minute_ago + ] + + # Check if under limit + if len(self.requests[key]) >= self.requests_per_minute: + return False + + # Add current request + self.requests[key].append(now) + return True + +class RateLimitMiddleware: + """Rate limiting middleware.""" + + def __init__(self, limiter: RateLimiter): + self.limiter = limiter + + async def __call__(self, request: Request, call_next): + """Process request with rate limiting.""" + # Use API key or IP address as key + api_key = request.headers.get("X-API-Key", request.client.host) + + if not self.limiter.is_allowed(api_key): + raise HTTPException( + status_code=429, + detail="Too many requests" + ) + + return await call_next(request) + +# Integrate with gateway +rate_limiter = RateLimiter(requests_per_minute=60) +gateway.app.middleware("http")(RateLimitMiddleware(rate_limiter)) +``` + +### 4. Service Aggregation +Implement response aggregation from multiple services. + +```python +class ServiceAggregator: + """Aggregate responses from multiple services.""" + + def __init__(self, gateway: APIGateway): + self.gateway = gateway + + async def aggregate_user_data(self, user_id: str) -> Dict[str, Any]: + """Aggregate user data from multiple services.""" + tasks = [ + self.fetch_user_profile(user_id), + self.fetch_user_orders(user_id), + self.fetch_user_recommendations(user_id) + ] + + results = await asyncio.gather(*tasks, return_exceptions=True) + + return { + "user_id": user_id, + "profile": results[0] if not isinstance(results[0], Exception) else None, + "orders": results[1] if not isinstance(results[1], Exception) else None, + "recommendations": results[2] if not isinstance(results[2], Exception) else None + } + + async def fetch_user_profile(self, user_id: str) -> Dict[str, Any]: + """Fetch user profile.""" + service_url = self.gateway.services.get("users") + if not service_url: + raise ValueError("Users service not found") + + response = await self.gateway.client.get(f"{service_url}/users/{user_id}") + return response.json() + + async def fetch_user_orders(self, user_id: str) -> Dict[str, Any]: + """Fetch user orders.""" + service_url = self.gateway.services.get("orders") + if not service_url: + raise ValueError("Orders service not found") + + response = await self.gateway.client.get(f"{service_url}/orders/{user_id}") + return response.json() + + async def fetch_user_recommendations(self, user_id: str) -> Dict[str, Any]: + """Fetch user recommendations.""" + service_url = self.gateway.services.get("products") + if not service_url: + raise ValueError("Products service not found") + + response = await self.gateway.client.get(f"{service_url}/recommendations/{user_id}") + return response.json() + +# Add aggregation endpoint +aggregator = ServiceAggregator(gateway) + +@gateway.app.get("/aggregate/user/{user_id}") +async def aggregate_user_endpoint(user_id: str): + """Aggregate user data.""" + return await aggregator.aggregate_user_data(user_id) +``` + +### 5. Circuit Breaker Pattern +Implement circuit breaker for fault tolerance. + +```python +from enum import Enum +import asyncio + +class CircuitState(Enum): + CLOSED = "closed" # Normal operation + OPEN = "open" # Failing, reject requests + HALF_OPEN = "half_open" # Testing if service recovered + +class CircuitBreaker: + """Circuit breaker for service resilience.""" + + def __init__(self, failure_threshold: int = 5, timeout: int = 60): + self.failure_threshold = failure_threshold + self.timeout = timeout + self.failure_count = 0 + self.state = CircuitState.CLOSED + self.last_failure_time = None + + def record_success(self): + """Record successful request.""" + self.failure_count = 0 + if self.state == CircuitState.HALF_OPEN: + self.state = CircuitState.CLOSED + + def record_failure(self): + """Record failed request.""" + self.failure_count += 1 + self.last_failure_time = time.time() + + if self.failure_count >= self.failure_threshold: + self.state = CircuitState.OPEN + + def can_attempt(self) -> bool: + """Check if request can be attempted.""" + if self.state == CircuitState.CLOSED: + return True + + if self.state == CircuitState.OPEN: + if time.time() - self.last_failure_time > self.timeout: + self.state = CircuitState.HALF_OPEN + return True + return False + + return True # HALF_OPEN state + +class CircuitBreakerMiddleware: + """Circuit breaker middleware.""" + + def __init__(self): + self.circuit_breakers: Dict[str, CircuitBreaker] = {} + + def get_breaker(self, service_name: str) -> CircuitBreaker: + """Get or create circuit breaker for service.""" + if service_name not in self.circuit_breakers: + self.circuit_breakers[service_name] = CircuitBreaker() + return self.circuit_breakers[service_name] + + async def call_with_breaker(self, service_name: str, callable_func, *args, **kwargs): + """Call service with circuit breaker protection.""" + breaker = self.get_breaker(service_name) + + if not breaker.can_attempt(): + raise HTTPException( + status_code=503, + detail=f"Service '{service_name}' is temporarily unavailable" + ) + + try: + result = await callable_func(*args, **kwargs) + breaker.record_success() + return result + except Exception as e: + breaker.record_failure() + raise e + +# Usage +circuit_breaker = CircuitBreakerMiddleware() + +@gateway.app.get("/products/{product_id}") +async def get_product(product_id: str): + """Get product with circuit breaker protection.""" + async def fetch_product(): + service_url = gateway.services.get("products") + response = await gateway.client.get(f"{service_url}/products/{product_id}") + return response.json() + + return await circuit_breaker.call_with_breaker("products", fetch_product) +``` + +### 6. Configuration and Service Discovery +Implement service discovery and configuration management. + +```python +import consul + +class ServiceRegistry: + """Service registry using Consul.""" + + def __init__(self, consul_host: str = "localhost", consul_port: int = 8500): + self.consul = consul.Consul(host=consul_host, port=consul_port) + self.logger = logging.getLogger("ServiceRegistry") + + def register_service(self, service_name: str, service_id: str, address: str, port: int): + """Register service with Consul.""" + self.consul.agent.service.register( + name=service_name, + service_id=service_id, + address=address, + port=port, + check=consul.Check.http(f"http://{address}:{port}/health", interval="10s") + ) + self.logger.info(f"Registered service: {service_name} ({service_id})") + + def deregister_service(self, service_id: str): + """Deregister service from Consul.""" + self.consul.agent.service.deregister(service_id) + self.logger.info(f"Deregistered service: {service_id}") + + def discover_service(self, service_name: str) -> list: + """Discover service instances.""" + _, services = self.consul.health.service(service_name, passing=True) + + instances = [] + for service in services: + instance = { + "id": service["Service"]["ID"], + "address": service["Service"]["Address"], + "port": service["Service"]["Port"] + } + instances.append(instance) + + return instances + + def get_service_url(self, service_name: str) -> str: + """Get service URL (load balanced).""" + instances = self.discover_service(service_name) + if not instances: + raise ValueError(f"No healthy instances found for {service_name}") + + # Simple round-robin selection + instance = instances[0] # Could implement proper load balancing + return f"http://{instance['address']}:{instance['port']}" + +# Integrate service discovery with gateway +service_registry = ServiceRegistry() + +# Auto-discover and register services +async def refresh_services(): + """Refresh service registry.""" + service_names = ["users", "products", "orders"] + + for service_name in service_names: + try: + service_url = service_registry.get_service_url(service_name) + gateway.register_service(service_name, service_url) + except ValueError as e: + gateway.logger.warning(f"Service {service_name} not available: {str(e)}") + +# Schedule periodic refresh +# asyncio.create_task(periodic_refresh()) +``` + +## Constraints +- **Single Point of Failure**: API gateway can become a bottleneck - implement high availability +- **Performance**: Gateway adds latency - optimize routing and caching +- **Complexity**: Gateway logic can become complex - keep it focused +- **Scalability**: Design gateway to scale horizontally +- **Security**: Implement proper authentication and authorization +- **Monitoring**: Monitor gateway performance and service health + +## Expected Output +A robust API gateway that provides single entry point, authentication, rate limiting, service aggregation, and fault tolerance for microservices architecture. diff --git a/TRAE-Skills/architecture/Adapter_Pattern_TypeScript.md b/TRAE-Skills/architecture/Adapter_Pattern_TypeScript.md new file mode 100644 index 0000000000000000000000000000000000000000..8810eb08007906069478f8907bc3b93eb5229637 --- /dev/null +++ b/TRAE-Skills/architecture/Adapter_Pattern_TypeScript.md @@ -0,0 +1,87 @@ +# Skill: Adapter Pattern in TypeScript + +## Purpose +To allow incompatible interfaces to work together by wrapping an object in an adapter that translates its interface into one that a client expects. + +## When to Use +- When integrating a third-party library whose interface doesn't match your application's internal requirements. +- When you want to standardize multiple different implementations of a service (e.g., different payment gateways). +- When you need to provide a stable interface while the underlying dependency is subject to change. + +## Procedure + +### 1. Define the Target Interface +This is the interface your application expects to use. + +```typescript +// logger.interface.ts +export interface ILogger { + log(message: string): void; + error(message: string): void; +} +``` + +### 2. The Incompatible Service (Adaptee) +An external library or old code with a different interface. + +```typescript +// legacy-logger.ts +export class LegacyLogger { + printMessage(msg: string) { + console.log(`[LEGACY]: ${msg}`); + } + + reportFailure(err: string) { + console.error(`[LEGACY ERROR]: ${err}`); + } +} +``` + +### 3. Implement the Adapter +The adapter implements the `Target` interface and delegates work to the `Adaptee`. + +```typescript +// logger-adapter.ts +import { ILogger } from './logger.interface'; +import { LegacyLogger } from './legacy-logger'; + +export class LoggerAdapter implements ILogger { + private legacyLogger: LegacyLogger; + + constructor(legacyLogger: LegacyLogger) { + this.legacyLogger = legacyLogger; + } + + log(message: string): void { + // Translate the call + this.legacyLogger.printMessage(message); + } + + error(message: string): void { + // Translate the call + this.legacyLogger.reportFailure(message); + } +} +``` + +### 4. Usage in Client Code +The client only knows about the `ILogger` interface. + +```typescript +function app(logger: ILogger) { + logger.log("Application started"); +} + +const legacy = new LegacyLogger(); +const adapter = new LoggerAdapter(legacy); + +app(adapter); +``` + +## Constraints +- **Complexity**: Don't use the pattern if you can easily modify the original class to match the interface. +- **Performance**: While negligible, the extra layer of indirection adds a tiny overhead. +- **Single Responsibility**: The adapter should only focus on translation, not adding new business logic. + +## Expected Output +A wrapper class that successfully bridges two incompatible interfaces, allowing them to communicate without changing their existing code. diff --git a/TRAE-Skills/architecture/Authentication_Strategy_Selection.md b/TRAE-Skills/architecture/Authentication_Strategy_Selection.md new file mode 100644 index 0000000000000000000000000000000000000000..98248653cf9d8f4727a357f3e66f648041cfc02a --- /dev/null +++ b/TRAE-Skills/architecture/Authentication_Strategy_Selection.md @@ -0,0 +1,12 @@ +# Authentication Strategy Selection + +## Strategies +1. **Session-Based**: Server stores session ID in cookie/DB. Simple, stateful. Good for monoliths. +2. **JWT (Stateless)**: Server signs token, client stores it. Good for microservices/mobile. Harder to revoke. +3. **OAuth/OIDC**: Delegated auth (Login with Google). Best for user convenience and security. +4. **Passwordless**: Magic links, OTPs. Reduces friction. + +## Security Considerations +- Always use HTTPS. +- Store passwords using bcrypt/argon2. +- Use `httpOnly` and `secure` cookies for storage where possible to prevent XSS. diff --git a/TRAE-Skills/architecture/BFF_Pattern_Implementation.md b/TRAE-Skills/architecture/BFF_Pattern_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..244d7cf28d174531c386d3813a242a95247a6032 --- /dev/null +++ b/TRAE-Skills/architecture/BFF_Pattern_Implementation.md @@ -0,0 +1,76 @@ +# BFF Pattern Implementation (Backend for Frontend) + +## Overview +A BFF is a dedicated backend service for a specific frontend (e.g., Web BFF, Mobile BFF). + +## Problem +Different frontends have different data needs. A "one-size-fits-all" API might over-fetch for mobile or under-fetch for desktop. + +## Solution +- The BFF calls downstream microservices and aggregates/formats data specifically for its frontend. +- Handles authentication, caching, and protocol translation (e.g., gRPC to JSON). + +## When to use +- You have multiple client types (Web, iOS, Android) with significantly different UI requirements. + +## Procedure + +### 1. Define Client-Specific Requirements +Identify the unique data needs for each client (Web vs. Mobile). +- **Web**: Full dashboard, complex tables, many fields. +- **Mobile**: Minimalist view, paginated lists, essential fields only (to save bandwidth). + +### 2. Implementation: Aggregation Logic +The BFF calls multiple downstream services and merges the results. + +```typescript +// Web BFF - Express/TypeScript +router.get('/dashboard', async (req, res) => { + const [userProfile, recentOrders, notifications] = await Promise.all([ + userService.getProfile(req.userId), + orderService.getRecent(req.userId), + notificationService.getUnread(req.userId) + ]); + + res.json({ + user: { name: userProfile.name, avatar: userProfile.avatar }, + orders: recentOrders.map(o => ({ id: o.id, total: o.price })), + alerts: notifications.length + }); +}); +``` + +### 3. Protocol Translation +Convert internal protocols (like gRPC) to client-friendly JSON/REST or GraphQL. + +```typescript +// Translating gRPC to JSON +async function getProduct(id: string) { + const grpcResponse = await productGrpcClient.getProduct({ id }); + return { + id: grpcResponse.uuid, + name: grpcResponse.title, + price: grpcResponse.amount / 100 // Convert cents to dollars + }; +} +``` + +### 4. Client-Specific Authentication +Handle different auth flows (e.g., Session cookies for Web, JWT for Mobile). + +```typescript +// Mobile BFF - JWT Validation +app.use((req, res, next) => { + const token = req.headers.authorization; + if (!isValidMobileToken(token)) return res.status(401).send(); + next(); +}); +``` + +## Constraints +- **Avoid Business Logic**: Business logic should stay in the downstream microservices; the BFF is for **formatting and aggregation**. +- **Latency**: Parallelize downstream calls (`Promise.all`) to avoid "waterfall" latency. +- **Resilience**: Implement timeouts and circuit breakers for downstream dependencies. + +## Expected Output +A middleware service that optimizes communication between specific frontend clients and the backend ecosystem, reducing payload sizes and simplifying frontend logic. diff --git a/TRAE-Skills/architecture/CQRS_Implementation.md b/TRAE-Skills/architecture/CQRS_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..9db597ada9ca646689ee8bef3d75576235a8a3be --- /dev/null +++ b/TRAE-Skills/architecture/CQRS_Implementation.md @@ -0,0 +1,340 @@ +# Skill: CQRS (Command Query Responsibility Segregation) Implementation + +## Purpose +To separate read and write operations using CQRS pattern for scalability, performance, and flexibility. + +## When to Use +- When you have high read/write ratio discrepancies +- For complex business domains +- When you need different data models for reading and writing +- For event-driven architectures +- When scaling read-heavy applications + +## Procedure + +### 1. Define Commands and Queries +Separate command and query models. + +```typescript +// Command Models +interface CreateProductCommand { + type: 'CREATE_PRODUCT'; + name: string; + description: string; + price: number; + category: string; +} + +interface UpdateProductCommand { + type: 'UPDATE_PRODUCT'; + id: string; + name?: string; + description?: string; + price?: number; + category?: string; +} + +interface DeleteProductCommand { + type: 'DELETE_PRODUCT'; + id: string; +} + +type ProductCommand = CreateProductCommand | UpdateProductCommand | DeleteProductCommand; + +// Query Models +interface GetProductQuery { + id: string; +} + +interface ListProductsQuery { + category?: string; + minPrice?: number; + maxPrice?: number; + page?: number; + limit?: number; +} + +interface ProductDTO { + id: string; + name: string; + description: string; + price: number; + category: string; + createdAt: string; + updatedAt: string; +} +``` + +### 2. Command Handlers +Implement command handlers for write operations. + +```typescript +import { v4 as uuidv4 } from 'uuid'; + +interface ProductEvent { + id: string; + type: string; + aggregateId: string; + data: any; + timestamp: Date; +} + +class ProductCommandHandler { + private eventStore: ProductEvent[] = []; + + async handle(command: ProductCommand): Promise { + switch (command.type) { + case 'CREATE_PRODUCT': + return await this.createProduct(command); + case 'UPDATE_PRODUCT': + return await this.updateProduct(command); + case 'DELETE_PRODUCT': + return await this.deleteProduct(command); + default: + throw new Error('Unknown command type'); + } + } + + private async createProduct(command: CreateProductCommand): Promise { + const productId = uuidv4(); + + const event: ProductEvent = { + id: uuidv4(), + type: 'PRODUCT_CREATED', + aggregateId: productId, + data: { + name: command.name, + description: command.description, + price: command.price, + category: command.category + }, + timestamp: new Date() + }; + + this.eventStore.push(event); + await this.publishEvent(event); + + return productId; + } + + private async updateProduct(command: UpdateProductCommand): Promise { + const event: ProductEvent = { + id: uuidv4(), + type: 'PRODUCT_UPDATED', + aggregateId: command.id, + data: { + name: command.name, + description: command.description, + price: command.price, + category: command.category + }, + timestamp: new Date() + }; + + this.eventStore.push(event); + await this.publishEvent(event); + + return command.id; + } + + private async deleteProduct(command: DeleteProductCommand): Promise { + const event: ProductEvent = { + id: uuidv4(), + type: 'PRODUCT_DELETED', + aggregateId: command.id, + data: {}, + timestamp: new Date() + }; + + this.eventStore.push(event); + await this.publishEvent(event); + + return command.id; + } + + private async publishEvent(event: ProductEvent) { + // Publish to event bus (Kafka, Redis, etc.) + console.log('Publishing event:', event); + } +} +``` + +### 3. Query Handlers +Implement query handlers for read operations. + +```typescript +class ProductQueryHandler { + private productReadModel: Map = new Map(); + + async handleGetProduct(query: GetProductQuery): Promise { + return this.productReadModel.get(query.id) || null; + } + + async handleListProducts(query: ListProductsQuery): Promise<{ products: ProductDTO[], total: number }> { + let products = Array.from(this.productReadModel.values()); + + if (query.category) { + products = products.filter(p => p.category === query.category); + } + + if (query.minPrice !== undefined) { + products = products.filter(p => p.price >= query.minPrice); + } + + if (query.maxPrice !== undefined) { + products = products.filter(p => p.price <= query.maxPrice); + } + + const total = products.length; + const page = query.page || 1; + const limit = query.limit || 10; + const start = (page - 1) * limit; + const end = start + limit; + + return { + products: products.slice(start, end), + total + }; + } + + // Update read model from events + async applyEvent(event: ProductEvent) { + switch (event.type) { + case 'PRODUCT_CREATED': + this.productReadModel.set(event.aggregateId, { + id: event.aggregateId, + name: event.data.name, + description: event.data.description, + price: event.data.price, + category: event.data.category, + createdAt: event.timestamp.toISOString(), + updatedAt: event.timestamp.toISOString() + }); + break; + case 'PRODUCT_UPDATED': + const product = this.productReadModel.get(event.aggregateId); + if (product) { + this.productReadModel.set(event.aggregateId, { + ...product, + ...event.data, + updatedAt: event.timestamp.toISOString() + }); + } + break; + case 'PRODUCT_DELETED': + this.productReadModel.delete(event.aggregateId); + break; + } + } +} +``` + +### 4. API Controllers +Create controllers for commands and queries. + +```typescript +import express from 'express'; + +const app = express(); +app.use(express.json()); + +const commandHandler = new ProductCommandHandler(); +const queryHandler = new ProductQueryHandler(); + +// Command endpoints +app.post('/products', async (req, res) => { + try { + const command: CreateProductCommand = { + type: 'CREATE_PRODUCT', + name: req.body.name, + description: req.body.description, + price: req.body.price, + category: req.body.category + }; + + const productId = await commandHandler.handle(command); + res.status(201).json({ id: productId }); + } catch (error) { + res.status(400).json({ error: error.message }); + } +}); + +app.put('/products/:id', async (req, res) => { + try { + const command: UpdateProductCommand = { + type: 'UPDATE_PRODUCT', + id: req.params.id, + name: req.body.name, + description: req.body.description, + price: req.body.price, + category: req.body.category + }; + + await commandHandler.handle(command); + res.status(204).send(); + } catch (error) { + res.status(400).json({ error: error.message }); + } +}); + +app.delete('/products/:id', async (req, res) => { + try { + const command: DeleteProductCommand = { + type: 'DELETE_PRODUCT', + id: req.params.id + }; + + await commandHandler.handle(command); + res.status(204).send(); + } catch (error) { + res.status(400).json({ error: error.message }); + } +}); + +// Query endpoints +app.get('/products/:id', async (req, res) => { + try { + const query: GetProductQuery = { id: req.params.id }; + const product = await queryHandler.handleGetProduct(query); + + if (!product) { + return res.status(404).json({ error: 'Product not found' }); + } + + res.json(product); + } catch (error) { + res.status(400).json({ error: error.message }); + } +}); + +app.get('/products', async (req, res) => { + try { + const query: ListProductsQuery = { + category: req.query.category as string, + minPrice: req.query.minPrice ? Number(req.query.minPrice) : undefined, + maxPrice: req.query.maxPrice ? Number(req.query.maxPrice) : undefined, + page: req.query.page ? Number(req.query.page) : undefined, + limit: req.query.limit ? Number(req.query.limit) : undefined + }; + + const result = await queryHandler.handleListProducts(query); + res.json(result); + } catch (error) { + res.status(400).json({ error: error.message }); + } +}); + +app.listen(3000, () => { + console.log('CQRS Server running on port 3000'); +}); +``` + +## Best Practices +- **Separate Models**: Keep write and read models separate +- **Eventual Consistency**: Embrace eventual consistency for read models +- **Event Sourcing**: Combine with Event Sourcing for full history +- **Validation**: Validate commands before processing +- **Idempotency**: Make commands idempotent +- **Performance**: Optimize read models for queries (denormalize, use indexes) +- **Scalability**: Scale read and write sides independently +- **Monitoring**: Monitor sync between write and read models diff --git a/TRAE-Skills/architecture/CQRS_Pattern_Implementation.md b/TRAE-Skills/architecture/CQRS_Pattern_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..c5055d9d69c974a02d16c4ec5722212f21ef7ef3 --- /dev/null +++ b/TRAE-Skills/architecture/CQRS_Pattern_Implementation.md @@ -0,0 +1,89 @@ +# Skill: CQRS Pattern Implementation + +## Purpose +To separate read and write operations into different models, allowing each to be optimized independently for performance, scalability, and security. + +## When to Use +- In complex domains where the read model significantly differs from the write model. +- When high performance is required for reads (e.g., complex dashboards). +- When using Event Sourcing or distributed systems. + +## Procedure + +### 1. Define the Command (Write) +Commands represent the intent to change state. + +```typescript +// commands/create-user.command.ts +export class CreateUserCommand { + constructor( + public readonly email: string, + public readonly name: string + ) {} +} +``` + +### 2. Implement the Command Handler +The handler performs the actual state change. + +```typescript +// handlers/create-user.handler.ts +export class CreateUserHandler { + async handle(command: CreateUserCommand) { + // 1. Validate logic + // 2. Persist to "Write Database" + // 3. (Optional) Publish event to sync Read Model + console.log(`Creating user: ${command.email}`); + } +} +``` + +### 3. Define the Query (Read) +Queries represent the request to retrieve data. + +```typescript +// queries/get-user-stats.query.ts +export class GetUserStatsQuery { + constructor(public readonly userId: string) {} +} +``` + +### 4. Implement the Query Handler +The query handler fetches data from an optimized read model (e.g., a flattened SQL table or Redis). + +```typescript +// handlers/get-user-stats.handler.ts +export class GetUserStatsHandler { + async handle(query: GetUserStatsQuery) { + // 1. Query optimized "Read Database" + return { loginCount: 10, lastSeen: new Date() }; + } +} +``` + +### 5. Implementation: The Bus (Dispatcher) +A simple dispatcher to route commands/queries to their handlers. + +```typescript +export class MessageBus { + private handlers = new Map(); + + register(type: any, handler: any) { + this.handlers.set(type, handler); + } + + async execute(message: any) { + const handler = this.handlers.get(message.constructor); + if (!handler) throw new Error('No handler registered'); + return await handler.handle(message); + } +} +``` + +## Constraints +- **Separation**: Never modify data inside a Query. Never return data (except ID/Ack) from a Command. +- **Consistency**: Be aware of "Eventual Consistency" if the read and write databases are separate. +- **Complexity**: Do not use CQRS for simple CRUD applications. + +## Expected Output +An architecture where the application's side-effects (Commands) and data retrieval (Queries) are handled by distinct, optimized paths. diff --git a/TRAE-Skills/architecture/Clean_Architecture_Node.md b/TRAE-Skills/architecture/Clean_Architecture_Node.md new file mode 100644 index 0000000000000000000000000000000000000000..6d53f714f43dec1b77c8adef1ead3e2885cbb77c --- /dev/null +++ b/TRAE-Skills/architecture/Clean_Architecture_Node.md @@ -0,0 +1,96 @@ +# Skill: Clean Architecture in Node.js + +## Purpose +To decouple business logic from frameworks, databases, and external tools, making the application easier to test and maintain over time. + +## When to Use +- When building complex enterprise applications. +- When you want to ensure the business logic can survive changes in the database (e.g., SQL to NoSQL) or framework (e.g., Express to NestJS). +- When you need high testability of core business rules. + +## Procedure + +### 1. Structure the Directory (Layers) +Organize the code into four distinct layers: +- **Entities**: Enterprise business rules (Plain objects/classes). +- **Use Cases**: Application-specific business rules. +- **Interface Adapters**: Controllers, Gateways, Presenters. +- **Frameworks & Drivers**: Express, TypeORM, External APIs. + +### 2. Implementation: The Entity +```typescript +// src/domain/entities/User.ts +export class User { + constructor( + public readonly id: string, + public readonly email: string, + public readonly passwordHash: string + ) {} + + public validateEmail() { + return this.email.includes('@'); + } +} +``` + +### 3. Implementation: The Use Case +Use cases should only depend on abstractions (interfaces). + +```typescript +// src/application/use-cases/CreateUser.ts +import { User } from '../../domain/entities/User'; + +export interface UserRepository { + save(user: User): Promise; + findByEmail(email: string): Promise; +} + +export class CreateUser { + constructor(private userRepository: UserRepository) {} + + async execute(data: any) { + const user = new User(Date.now().toString(), data.email, data.password); + if (!user.validateEmail()) throw new Error('Invalid email'); + + await this.userRepository.save(user); + } +} +``` + +### 4. Implementation: The Adapter (Repository) +Implement the interface defined in the application layer. + +```typescript +// src/infrastructure/repositories/SqlUserRepository.ts +import { UserRepository } from '../../application/use-cases/CreateUser'; +import { User } from '../../domain/entities/User'; + +export class SqlUserRepository implements UserRepository { + async save(user: User): Promise { + // DB logic using Prisma/TypeORM + } + + async findByEmail(email: string): Promise { + // DB logic + return null; + } +} +``` + +### 5. Dependency Injection +Wire everything together in the outermost layer. + +```typescript +// src/main.ts +const userRepository = new SqlUserRepository(); +const createUserUseCase = new CreateUser(userRepository); +const userController = new UserController(createUserUseCase); +``` + +## Constraints +- **Dependency Rule**: Dependencies must only point **inwards** (Infrastructure -> Application -> Domain). +- **No Frameworks in Domain**: Domain entities should be pure TypeScript/JavaScript. +- **Testing**: Use cases should be tested with mocks of the repositories. + +## Expected Output +A decoupled codebase where the core logic is independent of the UI and Database choices. diff --git a/TRAE-Skills/architecture/Database_Selection_SQL_vs_NoSQL.md b/TRAE-Skills/architecture/Database_Selection_SQL_vs_NoSQL.md new file mode 100644 index 0000000000000000000000000000000000000000..0bf2defbdfb7e6cddee270a53d6a323d8fa49680 --- /dev/null +++ b/TRAE-Skills/architecture/Database_Selection_SQL_vs_NoSQL.md @@ -0,0 +1,15 @@ +# Database Selection (SQL vs NoSQL) + +## SQL (Relational) +- **Examples**: PostgreSQL, MySQL. +- **Use Case**: Structured data, complex relationships, transactions (ACID), reporting. +- **Schema**: Rigid, requires migrations. + +## NoSQL (Non-Relational) +- **Examples**: MongoDB (Document), Redis (Key-Value), Cassandra (Wide-Column). +- **Use Case**: Unstructured data, high write throughput, horizontal scaling, flexible schema. +- **Schema**: Flexible/Schema-less. + +## Selection +- Choose **SQL** by default for most business applications. +- Choose **NoSQL** for specific needs like caching (Redis), massive logs, or rapidly changing data structures. diff --git a/TRAE-Skills/architecture/Domain_Driven_Design_Basics.md b/TRAE-Skills/architecture/Domain_Driven_Design_Basics.md new file mode 100644 index 0000000000000000000000000000000000000000..70731448a1d88d8d70c1461e297bcd59efe7632c --- /dev/null +++ b/TRAE-Skills/architecture/Domain_Driven_Design_Basics.md @@ -0,0 +1,25 @@ +# Skill: Domain-Driven Design (DDD) Basics + +## Purpose +To model complex software. + +## When to Use +- Bounded contexts, aggregates, and repositories. +- When the specific requirement for Domain-Driven Design (DDD) Basics arises in the project. + +## Procedure +1. **Analysis**: Understand the requirements for Domain-Driven Design (DDD) Basics. +2. **Implementation**: + - Step 1: Initialize necessary configurations. + - Step 2: Implement the core logic for Domain-Driven Design (DDD) Basics. + - Step 3: Handle edge cases and errors. +3. **Verification**: + - Test the implementation. + - Ensure it meets the project standards. + +## Constraints +- Follow existing project coding standards. +- Ensure performance and security best practices are met. + +## Expected Output +A working implementation of Domain-Driven Design (DDD) Basics with documentation and tests. diff --git a/TRAE-Skills/architecture/Event_Driven_Architecture_Basics.md b/TRAE-Skills/architecture/Event_Driven_Architecture_Basics.md new file mode 100644 index 0000000000000000000000000000000000000000..48ce8274e328c2b13954f66e5694f2e8df939758 --- /dev/null +++ b/TRAE-Skills/architecture/Event_Driven_Architecture_Basics.md @@ -0,0 +1,18 @@ +# Event-Driven Architecture Basics + +## Overview +Services communicate by emitting and listening for events, rather than direct calls. + +## Components +- **Producer**: Emits an event (e.g., "OrderPlaced"). +- **Broker**: Routes events (e.g., RabbitMQ, Kafka, AWS SNS/SQS). +- **Consumer**: Reacts to the event (e.g., "SendEmail", "UpdateInventory"). + +## Benefits +- **Decoupling**: Producers don't know consumers. +- **Resilience**: If a consumer is down, events can be queued. +- **Scalability**: Add more consumers to process heavy loads. + +## Challenges +- Eventual consistency. +- Debugging/Tracing flow. diff --git a/TRAE-Skills/architecture/Event_Sourcing_Pattern.md b/TRAE-Skills/architecture/Event_Sourcing_Pattern.md new file mode 100644 index 0000000000000000000000000000000000000000..51b675944abc375c66903aa6ef06ae34b889a864 --- /dev/null +++ b/TRAE-Skills/architecture/Event_Sourcing_Pattern.md @@ -0,0 +1,237 @@ +# Skill: Event Sourcing Pattern + +## Purpose +To implement event sourcing architecture, where all changes to application state are stored as a sequence of events, enabling audit trails, temporal queries, and state reconstruction. + +## When to Use +- When you need a complete audit trail of all changes +- For systems that require temporal queries (what was the state at time X?) +- When building event-driven systems or microservices +- For complex business processes that benefit from event replay +- When you need to debug by replaying past events + +## Procedure + +### 1. Define Events +Define immutable events that represent state changes. + +```typescript +// events.ts +type UserEvent = + | { type: 'USER_CREATED', id: string, name: string, email: string, timestamp: number } + | { type: 'USER_UPDATED', id: string, name?: string, email?: string, timestamp: number } + | { type: 'USER_DELETED', id: string, timestamp: number }; + +interface EventStore { + append(aggregateId: string, events: UserEvent[]): Promise; + read(aggregateId: string): Promise; + readAll(): Promise; +} +``` + +### 2. Implement Event Store +Create an event store to persist events. + +```typescript +import { createClient } from 'redis'; + +class RedisEventStore implements EventStore { + private client: ReturnType; + + constructor() { + this.client = createClient(); + } + + async connect() { + await this.client.connect(); + } + + async append(aggregateId: string, events: UserEvent[]): Promise { + const key = `events:${aggregateId}`; + for (const event of events) { + await this.client.rPush(key, JSON.stringify(event)); + } + // Publish event for subscribers + for (const event of events) { + await this.client.publish('events', JSON.stringify(event)); + } + } + + async read(aggregateId: string): Promise { + const key = `events:${aggregateId}`; + const eventStrings = await this.client.lRange(key, 0, -1); + return eventStrings.map(str => JSON.parse(str)); + } + + async readAll(): Promise { + const keys = await this.client.keys('events:*'); + const allEvents: UserEvent[] = []; + for (const key of keys) { + const eventStrings = await this.client.lRange(key, 0, -1); + allEvents.push(...eventStrings.map(str => JSON.parse(str))); + } + return allEvents.sort((a, b) => a.timestamp - b.timestamp); + } +} +``` + +### 3. Aggregate with Event Sourcing +Create an aggregate that reconstructs state from events. + +```typescript +interface User { + id: string; + name: string; + email: string; + deleted: boolean; +} + +class UserAggregate { + private user: User | null = null; + private pendingEvents: UserEvent[] = []; + + constructor(private eventStore: EventStore) {} + + async load(id: string): Promise { + const events = await this.eventStore.read(id); + for (const event of events) { + this.applyEvent(event); + } + } + + private applyEvent(event: UserEvent): void { + switch (event.type) { + case 'USER_CREATED': + this.user = { + id: event.id, + name: event.name, + email: event.email, + deleted: false, + }; + break; + case 'USER_UPDATED': + if (this.user) { + if (event.name) this.user.name = event.name; + if (event.email) this.user.email = event.email; + } + break; + case 'USER_DELETED': + if (this.user) { + this.user.deleted = true; + } + break; + } + } + + create(id: string, name: string, email: string): void { + if (this.user) throw new Error('User already exists'); + const event: UserEvent = { + type: 'USER_CREATED', + id, + name, + email, + timestamp: Date.now(), + }; + this.applyEvent(event); + this.pendingEvents.push(event); + } + + update(name?: string, email?: string): void { + if (!this.user || this.user.deleted) throw new Error('User not found or deleted'); + const event: UserEvent = { + type: 'USER_UPDATED', + id: this.user.id, + name, + email, + timestamp: Date.now(), + }; + this.applyEvent(event); + this.pendingEvents.push(event); + } + + delete(): void { + if (!this.user || this.user.deleted) throw new Error('User not found or deleted'); + const event: UserEvent = { + type: 'USER_DELETED', + id: this.user.id, + timestamp: Date.now(), + }; + this.applyEvent(event); + this.pendingEvents.push(event); + } + + async save(): Promise { + if (this.user && this.pendingEvents.length > 0) { + await this.eventStore.append(this.user.id, this.pendingEvents); + this.pendingEvents = []; + } + } + + getState(): User | null { + return this.user; + } +} +``` + +### 4. Projections (CQRS) +Build read models from events. + +```typescript +class UserProjection { + private users: Map = new Map(); + + constructor(private eventStore: EventStore) {} + + async rebuild(): Promise { + this.users.clear(); + const events = await this.eventStore.readAll(); + for (const event of events) { + this.apply(event); + } + } + + private apply(event: UserEvent): void { + switch (event.type) { + case 'USER_CREATED': + this.users.set(event.id, { + id: event.id, + name: event.name, + email: event.email, + deleted: false, + }); + break; + case 'USER_UPDATED': + const user = this.users.get(event.id); + if (user) { + if (event.name) user.name = event.name; + if (event.email) user.email = event.email; + } + break; + case 'USER_DELETED': + const deletedUser = this.users.get(event.id); + if (deletedUser) { + deletedUser.deleted = true; + } + break; + } + } + + getById(id: string): User | undefined { + return this.users.get(id); + } + + getAll(): User[] { + return Array.from(this.users.values()).filter(u => !u.deleted); + } +} +``` + +## Best Practices +- **Immutability**: Events should be immutable and never modified +- **Idempotency**: Applying events multiple times should have the same effect +- **Versioning**: Version events to handle schema changes +- **Snapshots**: Use snapshots to optimize loading large aggregates +- **Error Handling**: Handle concurrency with optimistic locking +- **Observability**: Monitor event store performance and consistency +- **Backup**: Regularly back up the event store +- **Testing**: Test event replay and aggregate behavior thoroughly diff --git a/TRAE-Skills/architecture/Frontend_Backend_Communication_Patterns.md b/TRAE-Skills/architecture/Frontend_Backend_Communication_Patterns.md new file mode 100644 index 0000000000000000000000000000000000000000..e47841b2b95e2cb52e6202aee4d0a39bcdec25d9 --- /dev/null +++ b/TRAE-Skills/architecture/Frontend_Backend_Communication_Patterns.md @@ -0,0 +1,11 @@ +# Frontend-Backend Communication Patterns + +## Overview +How the client and server exchange data. + +## Patterns +1. **REST**: Resource-based, standard HTTP verbs. Good for caching and simplicity. +2. **GraphQL**: Client specifies data requirements. Good for complex relational data and minimizing over-fetching. +3. **WebSockets**: Full-duplex communication. Essential for real-time chat, notifications. +4. **gRPC**: High-performance RPC using Protocol Buffers. Good for microservices. +5. **Server-Sent Events (SSE)**: Server pushes updates to client (one-way). diff --git a/TRAE-Skills/architecture/Microservices_vs_Monolith_Decision.md b/TRAE-Skills/architecture/Microservices_vs_Monolith_Decision.md new file mode 100644 index 0000000000000000000000000000000000000000..faf22b6346131a22bf64640eaff0cba77d9108ef --- /dev/null +++ b/TRAE-Skills/architecture/Microservices_vs_Monolith_Decision.md @@ -0,0 +1,12 @@ +# Microservices vs Monolith Decision + +## Monolith +- **Pros**: Simple deployment, easier debugging, shared memory/code, no network latency between calls. +- **Cons**: Harder to scale specific parts, tight coupling, long build times. + +## Microservices +- **Pros**: Independent scaling, technology agnostic per service, fault isolation. +- **Cons**: Distributed system complexity (network failures, consistency), operational overhead, monitoring challenges. + +## Rule of Thumb +Start with a **Modular Monolith**. Only split into microservices when you have a specific scaling problem or organizational need (independent teams) that justifies the complexity. diff --git a/TRAE-Skills/architecture/Multi_Tenancy_Architecture.md b/TRAE-Skills/architecture/Multi_Tenancy_Architecture.md new file mode 100644 index 0000000000000000000000000000000000000000..a08e048aac78d04e0ec01dd0e8431962323e44ee --- /dev/null +++ b/TRAE-Skills/architecture/Multi_Tenancy_Architecture.md @@ -0,0 +1,64 @@ +# Multi-Tenancy Architecture + +## Overview +Serving multiple customers (tenants) from a single instance of the application. + +## Procedure + +### 1. Choosing an Isolation Level +Select based on compliance, budget, and performance needs. +- **Shared Database, Shared Schema**: Use a `tenant_id` column on all tables. (Cheapest, easiest to scale). +- **Shared Database, Separate Schema**: Use Postgres schemas or separate DBs on the same server. +- **Separate Database**: High isolation for enterprise clients. + +### 2. Implementation: Middleware for Tenant Identification +Identify the tenant from the request (Subdomain, Header, or JWT claim). + +```typescript +// Express Middleware +export const tenantContext = (req, res, next) => { + const tenantId = req.headers['x-tenant-id'] || req.subdomains[0]; + if (!tenantId) return res.status(400).send('Tenant not identified'); + + // Store in AsyncLocalStorage or attach to req + req.tenantId = tenantId; + next(); +}; +``` + +### 3. Implementation: Scoped Database Queries +Ensure all queries are automatically filtered by the current `tenant_id`. + +```typescript +// Example using Prisma Middleware +prisma.$use(async (params, next) => { + if (['User', 'Post', 'Order'].includes(params.model)) { + if (params.action === 'findUnique' || params.action === 'findFirst') { + params.action = 'findFirst'; // findUnique doesn't support additional filters + params.args.where['tenantId'] = currentTenantId(); + } + if (params.action === 'findMany') { + params.args.where = { ...params.args.where, tenantId: currentTenantId() }; + } + } + return next(params); +}); +``` + +### 4. Handling Migrations +Ensure migrations run across all tenant schemas if using the "Schema per Tenant" approach. + +```bash +# Pseudocode for running migrations across schemas +for schema in $(get_all_tenants); do + DB_SCHEMA=$schema npm run migrate +done +``` + +## Constraints +- **Data Leakage**: The biggest risk in shared-schema multi-tenancy. Always use automated filters (RLS in Postgres or ORM middleware). +- **Noisy Neighbors**: One tenant's heavy usage shouldn't crash the app for others. Use rate limiting per tenant. +- **Global Data**: Decide how to handle shared data (e.g., a global `Country` list) that doesn't belong to any specific tenant. + +## Expected Output +A system architecture that allows scaling to thousands of customers while keeping their data logically or physically separated. diff --git a/TRAE-Skills/architecture/REST_vs_GraphQL_Selection.md b/TRAE-Skills/architecture/REST_vs_GraphQL_Selection.md new file mode 100644 index 0000000000000000000000000000000000000000..4cbe26d6a09a442d3bf7081e5b466277fd97e95a --- /dev/null +++ b/TRAE-Skills/architecture/REST_vs_GraphQL_Selection.md @@ -0,0 +1,15 @@ +# REST vs GraphQL Selection + +## Comparison + +| Feature | REST | GraphQL | +| :--- | :--- | :--- | +| **Data Fetching** | Fixed endpoints, potential over/under-fetching | Client specifies exact fields | +| **Caching** | HTTP caching is native and easy | Requires application-level caching (e.g., normalized cache) | +| **Versioning** | V1, V2 endpoints | Schema evolution (deprecation) | +| **Tooling** | Mature, simple (curl, browser) | Rich ecosystem (Apollo, Relay), requires introspection | +| **Complexity** | Lower setup complexity | Higher setup, schema maintenance | + +## Recommendation +- **Use REST** for simple apps, public APIs, or when HTTP caching is critical. +- **Use GraphQL** for complex data graphs, mobile apps (bandwidth constrained), or rapid frontend iteration. diff --git a/TRAE-Skills/architecture/SSR_vs_CSR_Decision_Matrix.md b/TRAE-Skills/architecture/SSR_vs_CSR_Decision_Matrix.md new file mode 100644 index 0000000000000000000000000000000000000000..53353f4bed120cd7d897ce6840835009ec721c57 --- /dev/null +++ b/TRAE-Skills/architecture/SSR_vs_CSR_Decision_Matrix.md @@ -0,0 +1,17 @@ +# SSR vs CSR Decision Matrix + +## Concepts +- **CSR (Client-Side Rendering)**: Browser downloads JS bundle and renders UI (e.g., CRA). +- **SSR (Server-Side Rendering)**: Server renders HTML and sends to browser (e.g., Next.js). +- **SSG (Static Site Generation)**: HTML generated at build time. + +## Decision Factors +- **SEO**: SSR/SSG is better for crawlers. +- **Initial Load**: SSR is faster (First Contentful Paint). +- **Interactivity**: CSR can be faster after initial load. +- **Cost**: SSR requires running a Node server (computational cost). + +## Verdict +- **Public Marketing Site/Blog**: SSG. +- **Dashboard/SaaS App**: CSR is often sufficient. +- **E-commerce/Social Network**: SSR for SEO and performance. diff --git a/TRAE-Skills/architecture/Saga_Pattern_Distributed_Transactions.md b/TRAE-Skills/architecture/Saga_Pattern_Distributed_Transactions.md new file mode 100644 index 0000000000000000000000000000000000000000..5b484a99c472a2aaf9f8213073549b814952d0b0 --- /dev/null +++ b/TRAE-Skills/architecture/Saga_Pattern_Distributed_Transactions.md @@ -0,0 +1,534 @@ +# Skill: Saga Pattern for Distributed Transactions + +## Purpose +To manage data consistency across multiple microservices without using two-phase commit, using a sequence of local transactions coordinated through events. + +## When to Use +- When implementing distributed transactions across microservices +- When you need eventual consistency rather than immediate consistency +- When long-running transactions span multiple services +- When you need compensating transactions for rollback + +## Procedure + +### 1. Saga Pattern Foundation +Implement the basic saga pattern structure. + +```python +from abc import ABC, abstractmethod +from typing import List, Any, Dict +import logging +from enum import Enum + +class SagaStatus(Enum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + COMPENSATING = "compensating" + COMPENSATED = "compensated" + +class SagaStep(ABC): + """Abstract base class for saga steps.""" + + @abstractmethod + async def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Execute the step.""" + pass + + @abstractmethod + async def compensate(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Compensate the step.""" + pass + +class Saga: + """Saga orchestrator for distributed transactions.""" + + def __init__(self, name: str, steps: List[SagaStep]): + self.name = name + self.steps = steps + self.status = SagaStatus.PENDING + self.current_step = 0 + self.context = {} + self.logger = logging.getLogger(f"Saga.{name}") + + async def execute(self, initial_context: Dict[str, Any] = None) -> Dict[str, Any]: + """Execute the saga.""" + self.context = initial_context or {} + self.status = SagaStatus.RUNNING + self.logger.info(f"Starting saga: {self.name}") + + try: + # Execute each step + for i, step in enumerate(self.steps): + self.current_step = i + self.logger.info(f"Executing step {i + 1}/{len(self.steps)}: {step.__class__.__name__}") + + step_result = await step.execute(self.context) + self.context.update(step_result) + + # All steps completed successfully + self.status = SagaStatus.COMPLETED + self.logger.info(f"Saga completed successfully: {self.name}") + + return self.context + + except Exception as e: + self.logger.error(f"Saga failed at step {self.current_step}: {str(e)}") + self.status = SagaStatus.FAILED + + # Start compensation + await self._compensate() + + raise + + async def _compensate(self): + """Compensate completed steps.""" + self.status = SagaStatus.COMPENSATING + self.logger.info(f"Starting compensation for saga: {self.name}") + + # Compensate in reverse order + for i in range(self.current_step - 1, -1, -1): + step = self.steps[i] + self.logger.info(f"Compensating step {i + 1}: {step.__class__.__name__}") + + try: + compensate_result = await step.compensate(self.context) + self.context.update(compensate_result) + except Exception as e: + self.logger.error(f"Compensation failed at step {i}: {str(e)}") + + self.status = SagaStatus.COMPENSATED + self.logger.info(f"Saga compensation completed: {self.name}") +``` + +### 2. Order Processing Saga Example +Implement a concrete saga for order processing. + +```python +import httpx +from typing import Optional + +class CreateOrderStep(SagaStep): + """Create order step.""" + + def __init__(self, order_data: Dict[str, Any]): + self.order_data = order_data + self.order_id: Optional[str] = None + self.logger = logging.getLogger("CreateOrderStep") + + async def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Create the order.""" + async with httpx.AsyncClient() as client: + response = await client.post( + "http://orders-service/api/orders", + json=self.order_data + ) + response.raise_for_status() + order = response.json() + + self.order_id = order["id"] + self.logger.info(f"Created order: {self.order_id}") + + return {"order_id": self.order_id, "order": order} + + async def compensate(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Cancel the order.""" + if self.order_id: + async with httpx.AsyncClient() as client: + response = await client.delete( + f"http://orders-service/api/orders/{self.order_id}" + ) + response.raise_for_status() + + self.logger.info(f"Cancelled order: {self.order_id}") + + return {"order_cancelled": True} + +class ReserveInventoryStep(SagaStep): + """Reserve inventory step.""" + + def __init__(self, items: List[Dict[str, Any]]): + self.items = items + self.reservation_id: Optional[str] = None + self.logger = logging.getLogger("ReserveInventoryStep") + + async def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Reserve inventory.""" + order_id = context.get("order_id") + + async with httpx.AsyncClient() as client: + response = await client.post( + "http://inventory-service/api/reservations", + json={ + "order_id": order_id, + "items": self.items + } + ) + response.raise_for_status() + reservation = response.json() + + self.reservation_id = reservation["id"] + self.logger.info(f"Reserved inventory: {self.reservation_id}") + + return {"reservation_id": self.reservation_id} + + async def compensate(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Release inventory reservation.""" + if self.reservation_id: + async with httpx.AsyncClient() as client: + response = await client.delete( + f"http://inventory-service/api/reservations/{self.reservation_id}" + ) + response.raise_for_status() + + self.logger.info(f"Released inventory: {self.reservation_id}") + + return {"inventory_released": True} + +class ProcessPaymentStep(SagaStep): + """Process payment step.""" + + def __init__(self, payment_data: Dict[str, Any]): + self.payment_data = payment_data + self.payment_id: Optional[str] = None + self.logger = logging.getLogger("ProcessPaymentStep") + + async def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Process payment.""" + order_id = context.get("order_id") + + async with httpx.AsyncClient() as client: + response = await client.post( + "http://payment-service/api/payments", + json={ + "order_id": order_id, + **self.payment_data + } + ) + response.raise_for_status() + payment = response.json() + + self.payment_id = payment["id"] + self.logger.info(f"Processed payment: {self.payment_id}") + + return {"payment_id": self.payment_id} + + async def compensate(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Refund payment.""" + if self.payment_id: + async with httpx.AsyncClient() as client: + response = await client.post( + f"http://payment-service/api/payments/{self.payment_id}/refund" + ) + response.raise_for_status() + + self.logger.info(f"Refunded payment: {self.payment_id}") + + return {"payment_refunded": True} + +class ConfirmOrderStep(SagaStep): + """Confirm order step.""" + + def __init__(self): + self.logger = logging.getLogger("ConfirmOrderStep") + + async def execute(self, context: Dict[str, Any]) -> Dict[str, Any]: + """Confirm the order.""" + order_id = context.get("order_id") + + async with httpx.AsyncClient() as client: + response = await client.put( + f"http://orders-service/api/orders/{order_id}/confirm" + ) + response.raise_for_status() + + self.logger.info(f"Confirmed order: {order_id}") + + return {"order_confirmed": True} + + async def compensate(self, context: Dict[str, Any]) -> Dict[str, Any]: + """No compensation needed for confirmation.""" + return {} +``` + +### 3. Saga Orchestration +Implement saga orchestration and execution. + +```python +class OrderProcessingSaga: + """Order processing saga orchestrator.""" + + def __init__(self, order_data: Dict[str, Any], items: List[Dict[str, Any]], payment_data: Dict[str, Any]): + self.saga = Saga( + name="OrderProcessing", + steps=[ + CreateOrderStep(order_data), + ReserveInventoryStep(items), + ProcessPaymentStep(payment_data), + ConfirmOrderStep() + ] + ) + self.logger = logging.getLogger("OrderProcessingSaga") + + async def process(self) -> Dict[str, Any]: + """Process the order saga.""" + try: + result = await self.saga.execute() + self.logger.info("Order processing completed successfully") + return { + "status": "completed", + "order_id": result.get("order_id"), + "payment_id": result.get("payment_id"), + "reservation_id": result.get("reservation_id") + } + except Exception as e: + self.logger.error(f"Order processing failed: {str(e)}") + return { + "status": "failed", + "error": str(e), + "saga_status": self.saga.status.value + } + +# Usage +async def process_order(): + """Process an order using saga pattern.""" + order_data = { + "customer_id": "customer-123", + "shipping_address": "123 Main St", + "total_amount": 99.99 + } + + items = [ + {"product_id": "product-1", "quantity": 2}, + {"product_id": "product-2", "quantity": 1} + ] + + payment_data = { + "method": "credit_card", + "card_token": "tok_visa", + "amount": 99.99 + } + + saga = OrderProcessingSaga(order_data, items, payment_data) + result = await saga.process() + + return result +``` + +### 4. Saga Persistence +Implement saga persistence for recovery. + +```python +from datetime import datetime +import json + +class SagaRepository: + """Repository for saga persistence.""" + + def __init__(self, db_connection): + self.db = db_connection + self.logger = logging.getLogger("SagaRepository") + + async def save_saga(self, saga: Saga) -> str: + """Save saga state.""" + saga_data = { + "name": saga.name, + "status": saga.status.value, + "current_step": saga.current_step, + "context": json.dumps(saga.context), + "created_at": datetime.utcnow().isoformat(), + "updated_at": datetime.utcnow().isoformat() + } + + # In real implementation, save to database + # saga_id = await self.db.sagas.insert_one(saga_data) + saga_id = f"saga-{datetime.utcnow().timestamp()}" + + self.logger.info(f"Saved saga: {saga_id}") + return saga_id + + async def update_saga(self, saga_id: str, saga: Saga): + """Update saga state.""" + saga_data = { + "status": saga.status.value, + "current_step": saga.current_step, + "context": json.dumps(saga.context), + "updated_at": datetime.utcnow().isoformat() + } + + # In real implementation, update in database + # await self.db.sagas.update_one({"_id": saga_id}, {"$set": saga_data}) + + self.logger.info(f"Updated saga: {saga_id}") + + async def get_saga(self, saga_id: str) -> Optional[Dict[str, Any]]: + """Get saga state.""" + # In real implementation, retrieve from database + # saga_data = await self.db.sagas.find_one({"_id": saga_id}) + return None + + async def get_pending_sagas(self) -> List[Dict[str, Any]]: + """Get all pending sagas for recovery.""" + # In real implementation, query database + # pending_sagas = await self.db.sagas.find({"status": {"$in": ["running", "failed"]}}) + return [] + +class SagaRecoveryService: + """Service for recovering failed sagas.""" + + def __init__(self, repository: SagaRepository): + self.repository = repository + self.logger = logging.getLogger("SagaRecoveryService") + + async def recover_sagas(self): + """Recover failed or pending sagas.""" + pending_sagas = await self.repository.get_pending_sagas() + + for saga_data in pending_sagas: + saga_id = saga_data["_id"] + self.logger.info(f"Recovering saga: {saga_id}") + + try: + # Reconstruct saga from saved state + saga = self._reconstruct_saga(saga_data) + + # Continue execution + await saga.execute(json.loads(saga_data["context"])) + + except Exception as e: + self.logger.error(f"Failed to recover saga {saga_id}: {str(e)}") + + def _reconstruct_saga(self, saga_data: Dict[str, Any]) -> Saga: + """Reconstruct saga from saved state.""" + # Implementation depends on how you store and reconstruct sagas + pass +``` + +### 5. Event-Driven Saga Choreography +Implement choreography-based saga using events. + +```python +from fastapi import BackgroundTasks +from typing import Callable + +class SagaChoreography: + """Choreography-based saga using events.""" + + def __init__(self, event_bus): + self.event_bus = event_bus + self.logger = logging.getLogger("SagaChoreography") + + # Setup event handlers + self._setup_event_handlers() + + def _setup_event_handlers(self): + """Setup event handlers for saga choreography.""" + + @self.event_bus.subscribe("OrderCreated") + async def on_order_created(event: Dict[str, Any]): + """Handle order created event.""" + self.logger.info(f"Order created: {event['order_id']}") + + # Trigger inventory reservation + await self.event_bus.publish("ReserveInventory", { + "order_id": event["order_id"], + "items": event["items"] + }) + + @self.event_bus.subscribe("InventoryReserved") + async def on_inventory_reserved(event: Dict[str, Any]): + """Handle inventory reserved event.""" + self.logger.info(f"Inventory reserved: {event['reservation_id']}") + + # Trigger payment processing + await self.event_bus.publish("ProcessPayment", { + "order_id": event["order_id"], + "reservation_id": event["reservation_id"], + "amount": event["amount"] + }) + + @self.event_bus.subscribe("PaymentProcessed") + async def on_payment_processed(event: Dict[str, Any]): + """Handle payment processed event.""" + self.logger.info(f"Payment processed: {event['payment_id']}") + + # Trigger order confirmation + await self.event_bus.publish("ConfirmOrder", { + "order_id": event["order_id"], + "payment_id": event["payment_id"] + }) + + @self.event_bus.subscribe("PaymentFailed") + async def on_payment_failed(event: Dict[str, Any]): + """Handle payment failed event.""" + self.logger.error(f"Payment failed: {event['error']}") + + # Trigger compensation + await self.event_bus.publish("ReleaseInventory", { + "reservation_id": event["reservation_id"] + }) + + await self.event_bus.publish("CancelOrder", { + "order_id": event["order_id"] + }) + + @self.event_bus.subscribe("OrderConfirmed") + async def on_order_confirmed(event: Dict[str, Any]): + """Handle order confirmed event.""" + self.logger.info(f"Order confirmed: {event['order_id']}") + + # Notify customer + await self.event_bus.publish("SendOrderConfirmation", { + "order_id": event["order_id"] + }) + +class EventBus: + """Simple event bus for saga choreography.""" + + def __init__(self): + self.subscribers: Dict[str, List[Callable]] = {} + self.logger = logging.getLogger("EventBus") + + def subscribe(self, event_type: str) -> Callable: + """Decorator for subscribing to events.""" + def decorator(func: Callable): + if event_type not in self.subscribers: + self.subscribers[event_type] = [] + self.subscribers[event_type].append(func) + return func + return decorator + + async def publish(self, event_type: str, event_data: Dict[str, Any]): + """Publish event to subscribers.""" + if event_type in self.subscribers: + self.logger.info(f"Publishing event: {event_type}") + + for handler in self.subscribers[event_type]: + try: + await handler(event_data) + except Exception as e: + self.logger.error(f"Error handling event {event_type}: {str(e)}") + +# Usage +event_bus = EventBus() +saga_choreography = SagaChoreography(event_bus) + +# Start saga by publishing initial event +await event_bus.publish("CreateOrder", { + "customer_id": "customer-123", + "items": [{"product_id": "product-1", "quantity": 2}], + "total_amount": 99.99 +}) +``` + +## Constraints +- **Eventual Consistency**: Sagas provide eventual consistency, not immediate consistency +- **Complexity**: Compensation logic can be complex and error-prone +- **Timing**: Long-running sagas may hold resources for extended periods +- **Monitoring**: Requires comprehensive monitoring and logging +- **Testing**: Testing saga flows can be challenging +- **Scalability**: Event choreography can become complex with many services + +## Expected Output +A robust saga pattern implementation that manages distributed transactions across microservices with proper compensation, persistence, and recovery mechanisms. diff --git a/TRAE-Skills/architecture/Scaling_Strategies_Horizontal_vs_Vertical.md b/TRAE-Skills/architecture/Scaling_Strategies_Horizontal_vs_Vertical.md new file mode 100644 index 0000000000000000000000000000000000000000..425f91272f4c69dda47e428a34d63b8b13e48463 --- /dev/null +++ b/TRAE-Skills/architecture/Scaling_Strategies_Horizontal_vs_Vertical.md @@ -0,0 +1,14 @@ +# Scaling Strategies (Horizontal vs Vertical) + +## Vertical Scaling (Scale Up) +- **Action**: Add more CPU/RAM to the existing server. +- **Pros**: Simple, no code changes. +- **Cons**: Hardware limits, downtime to upgrade, single point of failure. + +## Horizontal Scaling (Scale Out) +- **Action**: Add more servers (nodes) behind a load balancer. +- **Pros**: Unlimited theoretical scale, redundancy/high availability. +- **Cons**: Complexity (statelessness required, distributed data consistency), cost overhead. + +## Strategy +Design for **Horizontal Scaling** (stateless apps) from the start, but **Vertically Scale** the database until it becomes a bottleneck, then shard or replicate. diff --git a/TRAE-Skills/architecture/Serverless_Architecture_Considerations.md b/TRAE-Skills/architecture/Serverless_Architecture_Considerations.md new file mode 100644 index 0000000000000000000000000000000000000000..79804463c9c59e0f06258b169617ea3c2d939fda --- /dev/null +++ b/TRAE-Skills/architecture/Serverless_Architecture_Considerations.md @@ -0,0 +1,15 @@ +# Serverless Architecture Considerations + +## Overview +Running code (Functions as a Service) without managing servers (e.g., AWS Lambda, Vercel Functions). + +## Pros +- **Cost**: Pay per execution. +- **Scaling**: Auto-scales to zero and up. +- **Ops**: minimal infrastructure management. + +## Cons +- **Cold Starts**: Latency on first request. +- **Stateless**: Database connections need management (e.g., connection pooling like AWS RDS Proxy). +- **Limits**: Execution time and memory limits. +- **Vendor Lock-in**: Harder to move to another provider. diff --git a/TRAE-Skills/architecture/Stack_Selection_Criteria.md b/TRAE-Skills/architecture/Stack_Selection_Criteria.md new file mode 100644 index 0000000000000000000000000000000000000000..bf141af5c31d4cfb9b14954ea9626f36105dbd0a --- /dev/null +++ b/TRAE-Skills/architecture/Stack_Selection_Criteria.md @@ -0,0 +1,14 @@ +# Stack Selection Criteria + +## Overview +Choosing the right technology stack is a critical early decision. + +## Factors to Consider +1. **Team Expertise**: What does the team already know? +2. **Project Requirements**: Real-time? High traffic? Mobile? +3. **Ecosystem & Community**: Library availability, support, hiring pool. +4. **Performance**: Speed, scalability, resource usage. +5. **Time to Market**: Rapid prototyping vs. long-term stability. + +## Decision Matrix +Create a weighted matrix to score options against these factors. diff --git a/TRAE-Skills/backend/API_REST_Endpoint_Design.md b/TRAE-Skills/backend/API_REST_Endpoint_Design.md new file mode 100644 index 0000000000000000000000000000000000000000..01fb97a02d2042a2d1ed9f56d6a2f34229e1520e --- /dev/null +++ b/TRAE-Skills/backend/API_REST_Endpoint_Design.md @@ -0,0 +1,39 @@ +# Skill: API REST Endpoint Design + +## Purpose +To design and implement a RESTful API endpoint that is consistent, scalable, and follows industry best practices for HTTP methods, status codes, and URL structure. + +## When to Use +- When creating a new API resource or sub-resource. +- When refactoring existing endpoints to meet REST standards. +- When defining the contract between frontend and backend. + +## Procedure +1. **Identify Resource**: Determine the noun representing the resource (e.g., `/users`, `/orders`). Use plural nouns. +2. **Select HTTP Method**: + - `GET` for retrieval. + - `POST` for creation. + - `PUT` for full replacement. + - `PATCH` for partial update. + - `DELETE` for removal. +3. **Define URL Structure**: + - Use hierarchy: `/resources/{id}/sub-resources`. + - Avoid verbs in URLs (e.g., use `POST /users/login` only if `session` is the resource, otherwise standard RPC might be an exception, but prefer REST). +4. **Define Request Payload**: + - Specify JSON body structure for POST/PUT/PATCH. + - Define query parameters for GET (filtering, pagination, sorting). +5. **Define Response Structure**: + - Standardize success response (e.g., `{ data: ... }`). + - Standardize error response (e.g., `{ error: { code: ..., message: ... } }`). +6. **Select Status Codes**: + - 200 (OK), 201 (Created), 204 (No Content). + - 400 (Bad Request), 401 (Unauthorized), 403 (Forbidden), 404 (Not Found). + - 500 (Internal Server Error). + +## Constraints +- Do NOT expose internal database IDs if UUIDs are available/preferred. +- Do NOT mix snake_case and camelCase in JSON responses (prefer camelCase for JS/Node). +- Ensure all endpoints are stateless. + +## Expected Output +A fully defined route handler (or controller method) with request validation, business logic invocation, and a standardized JSON response. diff --git a/TRAE-Skills/backend/API_Versioning_Strategies.md b/TRAE-Skills/backend/API_Versioning_Strategies.md new file mode 100644 index 0000000000000000000000000000000000000000..86da59712c9981ac8990da5625210a6d8523a0da --- /dev/null +++ b/TRAE-Skills/backend/API_Versioning_Strategies.md @@ -0,0 +1,53 @@ +# Skill: API Versioning Strategies + +## Purpose +To manage changes in API contracts without breaking existing clients, ensuring backward compatibility. + +## When to Use +- When introducing breaking changes (renaming fields, changing data types, removing endpoints). +- When supporting mobile apps that cannot be forced to update immediately. + +## Procedure + +### 1. URI Versioning (Most Common) +Include version in the path. Clear and cache-friendly. +- `GET /api/v1/users` +- `GET /api/v2/users` + +Implementation (Express): +```javascript +const v1Router = require('./routes/v1'); +const v2Router = require('./routes/v2'); + +app.use('/api/v1', v1Router); +app.use('/api/v2', v2Router); +``` + +### 2. Header Versioning +Cleaner URLs, but harder to test via browser. +- Header: `Accept-Version: v1` + +Implementation: +```javascript +app.get('/users', (req, res) => { + const version = req.get('Accept-Version'); + if (version === 'v2') return handlerV2(req, res); + return handlerV1(req, res); +}); +``` + +### 3. Query Parameter +Easy for debugging. +- `GET /api/users?version=1` + +### 4. Deprecation Strategy +1. **Announce**: Mark fields/endpoints as deprecated in docs. +2. **Sunset Header**: Add `Sunset: ` header to responses indicating when the version will be removed. +3. **Brownouts**: Intentionally fail requests for short periods as EOL approaches to alert developers. + +## Constraints +- **Code Duplication**: Managing multiple versions can lead to duplicated logic. Use "Controller-Service" pattern to share business logic where possible, applying adapters for the response format. +- **Maintenance**: Limit supported versions (e.g., maintain only the last 2 major versions). + +## Expected Output +A stable API ecosystem where clients can upgrade at their own pace without service interruption. diff --git a/TRAE-Skills/backend/Advanced_RBAC.md b/TRAE-Skills/backend/Advanced_RBAC.md new file mode 100644 index 0000000000000000000000000000000000000000..5b793a3114c163b0aa0c7f2b6fa75c85ecc05efc --- /dev/null +++ b/TRAE-Skills/backend/Advanced_RBAC.md @@ -0,0 +1,59 @@ +# Skill: Advanced RBAC (Role-Based Access Control) + +## Purpose +To implement a scalable permission system where users have roles, and roles have granular permissions. + +## When to Use +- B2B SaaS applications (Admin, Editor, Viewer). +- Systems with complex access requirements. + +## Procedure + +### 1. Data Model +- **Users**: `id`, `name`, `role_id` (or m:n `user_roles`). +- **Roles**: `id`, `name` (e.g., 'admin', 'editor'). +- **Permissions**: `id`, `action`, `resource` (e.g., `create:post`, `read:user`). +- **RolePermissions**: Join table. + +### 2. Middleware (Enforcement) +Don't check roles (`if user.role === 'admin'`). **Check permissions**. +```javascript +// middleware/authorize.js +const authorize = (requiredPermission) => { + return async (req, res, next) => { + const userPermissions = await getUserPermissions(req.user.id); // Cached in JWT or Redis + if (!userPermissions.includes(requiredPermission)) { + return res.status(403).json({ error: 'Forbidden' }); + } + next(); + }; +}; + +// usage +app.post('/posts', authorize('create:post'), createPost); +``` + +### 3. Hierarchy (Optional) +Roles can inherit from others. +- `Admin` has all permissions of `Editor` + `delete:user`. +- This logic is handled in the permission resolution function. + +### 4. Frontend Integration +Send permissions list to frontend on login. +```javascript +{ + token: "...", + user: { + id: 1, + permissions: ["read:posts", "create:posts"] + } +} +``` +Use a `` component to hide/show UI elements. + +## Constraints +- **Granularity**: Balance between too broad (`manage:all`) and too specific (`update:post:title`). +- **Caching**: Fetching permissions on every request is slow. Cache them in the JWT or Redis (invalidate on role change). + +## Expected Output +A flexible security model allowing admins to configure what users can do without changing code. diff --git a/TRAE-Skills/backend/Background_Jobs_BullMQ.md b/TRAE-Skills/backend/Background_Jobs_BullMQ.md new file mode 100644 index 0000000000000000000000000000000000000000..52122f394d7db5f9aed2bad3baed40680857e544 --- /dev/null +++ b/TRAE-Skills/backend/Background_Jobs_BullMQ.md @@ -0,0 +1,60 @@ +# Skill: Background Jobs (BullMQ) + +## Purpose +To offload time-consuming tasks (email sending, image processing, report generation) from the main HTTP request-response cycle using Redis-based queues. + +## When to Use +- When an API request takes longer than a few seconds. +- To handle rate-limited 3rd party API calls. +- To retry failed operations automatically. + +## Procedure + +### 1. Setup +Requires a running Redis instance. +```bash +npm install bullmq ioredis +``` + +### 2. Producer (Adding Jobs) +Add jobs to a queue. +```javascript +const { Queue } = require('bullmq'); +const emailQueue = new Queue('email-queue', { connection: redisConnection }); + +async function sendWelcomeEmail(user) { + await emailQueue.add('welcome-email', { + email: user.email, + name: user.name + }, { + attempts: 3, // Retry 3 times on failure + backoff: { type: 'exponential', delay: 1000 } + }); +} +``` + +### 3. Consumer (Processing Jobs) +Worker processes jobs from the queue. Run this in a separate process or service. +```javascript +const { Worker } = require('bullmq'); + +const worker = new Worker('email-queue', async job => { + console.log(`Sending email to ${job.data.email}`); + await sendEmailService(job.data); +}, { connection: redisConnection }); + +worker.on('completed', job => { + console.log(`Job ${job.id} completed`); +}); + +worker.on('failed', (job, err) => { + console.log(`Job ${job.id} failed: ${err.message}`); +}); +``` + +## Constraints +- **Redis Dependency**: The queue relies entirely on Redis. If Redis goes down, queues are inaccessible (though data persists if configured). +- **Concurrency**: Configure `concurrency` option in Worker to process multiple jobs in parallel based on CPU resources. + +## Expected Output +Improved API response times (immediate 202 Accepted) and reliable background task execution with retry mechanisms. diff --git a/TRAE-Skills/backend/Caching_Strategy_Redis.md b/TRAE-Skills/backend/Caching_Strategy_Redis.md new file mode 100644 index 0000000000000000000000000000000000000000..1597ce83fd86c96f2c66af2cedf8541923b598a8 --- /dev/null +++ b/TRAE-Skills/backend/Caching_Strategy_Redis.md @@ -0,0 +1,31 @@ +# Skill: Caching Strategy (Redis) + +## Purpose +To improve response times and reduce database load by storing frequently accessed data in a fast, in-memory store. + +## When to Use +- When fetching data that changes infrequently (e.g., product catalog, configuration). +- When computing expensive aggregations. +- When endpoints have high read-to-write ratios. + +## Procedure +1. **Identify Data**: Determine what to cache and for how long (TTL). +2. **Check Cache First**: + - Construct a unique cache key (e.g., `product:123`). + - Query Redis for the key. + - If found (Hit), return JSON parsed data immediately. +3. **Fetch on Miss**: + - If not found (Miss), query the primary database. + - Save the result to Redis with an expiration (`SETEX key ttl value`). + - Return the data. +4. **Invalidation Strategy**: + - Determine how to handle updates (Cache-Aside is common). + - When data is updated/deleted, delete the corresponding Redis key. + +## Constraints +- Handle cache stampedes (many requests for same missing key at once). +- Ensure data consistency (don't serve stale data longer than acceptable). +- Serialize/deserialize objects correctly (Redis stores strings/buffers). + +## Expected Output +A caching utility or middleware that transparently handles cache hits and misses, significantly reducing API latency. diff --git a/TRAE-Skills/backend/Database_Locking_Strategies.md b/TRAE-Skills/backend/Database_Locking_Strategies.md new file mode 100644 index 0000000000000000000000000000000000000000..e76c163f657b67a62ae4e07700012c612cb7196d --- /dev/null +++ b/TRAE-Skills/backend/Database_Locking_Strategies.md @@ -0,0 +1,51 @@ +# Skill: Database Locking Strategies + +## Purpose +To manage concurrent access to database records, preventing race conditions and data inconsistencies (e.g., double spending, overbooking). + +## When to Use +- Financial transactions. +- Inventory management. +- "Check-and-Set" operations. + +## Procedure + +### 1. Optimistic Locking (Versioning) +Assume conflict is rare. Use a `version` column. +1. Read record: `SELECT * FROM products WHERE id=1;` (Returns version: 5) +2. Update with check: + ```sql + UPDATE products SET stock = stock - 1, version = 6 + WHERE id = 1 AND version = 5; + ``` +3. Check result: If "0 rows affected", someone else modified it. Retry or throw error. + +**Implementation (TypeORM/Prisma)**: +Often supported natively (`@VersionColumn` in TypeORM). + +### 2. Pessimistic Locking +Assume conflict is likely. Lock the row for the transaction duration. +1. Start Transaction. +2. Select for Update: + ```sql + SELECT * FROM products WHERE id = 1 FOR UPDATE; + ``` + (This blocks other transactions trying to lock this row). +3. Perform Logic. +4. Commit (Release Lock). + +### 3. Advisory Locks (App Level) +Lock an arbitrary abstract name/ID, not a specific row. +```sql +-- PostgreSQL +SELECT pg_advisory_xact_lock(12345); +-- Do work... +-- Lock released automatically at end of transaction +``` + +## Constraints +- **Deadlocks**: Pessimistic locking can lead to deadlocks if two transactions lock resources in reverse order. Always acquire locks in a consistent order (e.g., sort by ID). +- **Performance**: `FOR UPDATE` reduces concurrency. Use Optimistic locking for high-traffic, low-conflict scenarios. + +## Expected Output +Data integrity guarantees under high concurrency, ensuring atomic transitions of state. diff --git a/TRAE-Skills/backend/Database_Schema_Migration.md b/TRAE-Skills/backend/Database_Schema_Migration.md new file mode 100644 index 0000000000000000000000000000000000000000..53925300cc43a3943cf0f2c125071c4905b0183f --- /dev/null +++ b/TRAE-Skills/backend/Database_Schema_Migration.md @@ -0,0 +1,30 @@ +# Skill: Database Schema Migration + +## Purpose +To safely modify the database schema (tables, columns, indexes) using version-controlled migration files, ensuring consistency across development, staging, and production environments. + +## When to Use +- When adding new tables or columns. +- When modifying column types or constraints. +- When adding indexes for performance. +- When seeding initial reference data. + +## Procedure +1. **Generate Migration File**: Use the ORM/Query Builder CLI (e.g., Knex, TypeORM, Prisma) to create a new migration file with a timestamp prefix. +2. **Implement `up` Method**: + - Define the changes to apply (e.g., `createTable`, `alterTable`). + - Ensure correct data types and constraints (NOT NULL, DEFAULT). + - Add foreign keys with appropriate `onDelete` behavior (CASCADE, SET NULL). +3. **Implement `down` Method**: + - Define the exact reverse of the `up` method. + - Ensure data loss warnings are considered if dropping tables/columns. +4. **Review SQL**: If possible, inspect the generated SQL to ensure it performs efficiently (e.g., avoiding table locks on large tables if possible). +5. **Test Migration**: Run `migrate:up` and `migrate:down` locally to verify reversibility. + +## Constraints +- Never modify an existing migration file that has already been committed/deployed. Create a new one. +- Avoid logic that depends on application code within migrations (migrations should be standalone SQL/schema logic). +- Ensure migrations are idempotent where possible. + +## Expected Output +A valid migration file in the project's migrations directory that successfully alters the database schema when run and reverts cleanly when rolled back. diff --git a/TRAE-Skills/backend/Database_Seeding.md b/TRAE-Skills/backend/Database_Seeding.md new file mode 100644 index 0000000000000000000000000000000000000000..3aa53740ce606dfd0007425069ccaaf09fd33e18 --- /dev/null +++ b/TRAE-Skills/backend/Database_Seeding.md @@ -0,0 +1,28 @@ +# Skill: Database Seeding + +## Purpose +To populate the database with predictable, realistic initial data for development, testing, and staging environments. + +## When to Use +- When setting up a new developer machine. +- When running integration tests (resetting state). +- When initializing a new environment with lookup data (e.g., roles, categories). + +## Procedure +1. **Select Library**: Use a faker library (e.g., `@faker-js/faker`) to generate random realistic data. +2. **Define Factories**: Create functions that generate a single valid object for each entity (e.g., `makeUser()`). +3. **Define Seeder Script**: + - Connect to DB. + - Clear existing data (optional/careful). + - Loop and insert data using factories. + - Maintain referential integrity (create Users first, then Posts). +4. **Idempotency**: Check if data exists before inserting (if running on persistent envs). +5. **Run Command**: Add `npm run seed` script. + +## Constraints +- NEVER run destructive seeds against production databases (add safety checks). +- Ensure seeded data passes all validation rules. +- Keep seed execution time reasonable. + +## Expected Output +A seed script that, when executed, populates the database with a rich set of test data. diff --git a/TRAE-Skills/backend/Distributed_Transactions_Sagas.md b/TRAE-Skills/backend/Distributed_Transactions_Sagas.md new file mode 100644 index 0000000000000000000000000000000000000000..f7f63c61630841d928931af83f91540a7dbb4097 --- /dev/null +++ b/TRAE-Skills/backend/Distributed_Transactions_Sagas.md @@ -0,0 +1,42 @@ +# Skill: Distributed Transactions (Sagas) + +## Purpose +To maintain data consistency across multiple microservices where local ACID transactions are not possible. + +## When to Use +- E-commerce order: Payment Service -> Inventory Service -> Shipping Service. +- Any operation spanning multiple databases. + +## Procedure + +### 1. The Saga Pattern +A sequence of local transactions. If one fails, execute **Compensating Transactions** to undo changes. + +### 2. Choreography (Event-Based) +Services listen to events and trigger the next step. +1. **Order Service**: Creates Order (Pending). Publishes `OrderCreated`. +2. **Payment Service**: Listens to `OrderCreated`. Charges card. Publishes `PaymentProcessed` or `PaymentFailed`. +3. **Inventory Service**: Listens to `PaymentProcessed`. Reserves stock. Publishes `StockReserved`. +4. **Order Service**: Listens to `StockReserved`. Updates Order to `Confirmed`. + +**Compensation**: +- If Payment fails: Publish `PaymentFailed`. +- Order Service listens: Cancels Order. + +### 3. Orchestration (Controller-Based) +A central "Saga Orchestrator" tells services what to do. +1. **Orchestrator**: Send "Reserve Stock" command to Inventory. +2. **Inventory**: Responds "Success". +3. **Orchestrator**: Send "Charge Payment" command. +4. **Payment**: Responds "Insufficient Funds". +5. **Orchestrator**: Send "Release Stock" command (Compensation) to Inventory. + +### 4. Implementation +Use frameworks like Temporal.io or custom state machines with message queues (BullMQ/Kafka). + +## Constraints +- **Eventual Consistency**: The system is not consistent at every moment. UI must handle "Pending" states. +- **Isolation**: Sagas lack Isolation (I in ACID). "Dirty reads" can happen. Use semantic locking (e.g., state=PENDING_APPROVAL) to prevent other ops from touching the entity. + +## Expected Output +A resilient distributed workflow that guarantees the system eventually reaches a consistent state (either Success or Rolled Back). diff --git a/TRAE-Skills/backend/Elasticsearch_Integration.md b/TRAE-Skills/backend/Elasticsearch_Integration.md new file mode 100644 index 0000000000000000000000000000000000000000..0b2f1153b83c291ac657115af70bf336601b38ff --- /dev/null +++ b/TRAE-Skills/backend/Elasticsearch_Integration.md @@ -0,0 +1,71 @@ +# Skill: Elasticsearch Integration + +## Purpose +To implement full-text search, autocomplete, and complex filtering capabilities that standard SQL databases cannot handle efficiently. + +## When to Use +- Search bars (e-commerce products, user finder). +- Log analysis. +- Fuzzy matching (handling typos). + +## Procedure + +### 1. Connection +```javascript +const { Client } = require('@elastic/elasticsearch'); +const client = new Client({ node: 'http://localhost:9200' }); +``` + +### 2. Indexing Data (Sync) +Keep Elasticsearch in sync with the primary DB. +- **On Save**: When User is updated in SQL, update Elasticsearch. +```javascript +await client.index({ + index: 'users', + id: user.id, + document: { + name: user.name, + email: user.email, + bio: user.bio + } +}); +``` + +### 3. Searching +Perform a multi-match fuzzy search. +```javascript +const result = await client.search({ + index: 'users', + query: { + multi_match: { + query: 'jonh doe', // typo intended + fields: ['name', 'bio'], + fuzziness: 'AUTO' + } + } +}); +const hits = result.hits.hits.map(h => h._source); +``` + +### 4. Mappings +Define types explicitly for better performance. +```javascript +await client.indices.create({ + index: 'users', + mappings: { + properties: { + name: { type: 'text' }, // Full-text search + email: { type: 'keyword' }, // Exact match only + created_at: { type: 'date' } + } + } +}); +``` + +## Constraints +- **Sync Latency**: Data might not be immediately available for search (near real-time). +- **Cost**: Elasticsearch is resource-heavy (RAM/CPU). +- **Reindexing**: Changing mappings often requires reindexing all data. + +## Expected Output +Fast, relevant search results with capabilities like highlighting, facets, and fuzzy matching. diff --git a/TRAE-Skills/backend/Error_Handling_Express.md b/TRAE-Skills/backend/Error_Handling_Express.md new file mode 100644 index 0000000000000000000000000000000000000000..06b9a763af5e8f1155c66b1b543bbf547312720f --- /dev/null +++ b/TRAE-Skills/backend/Error_Handling_Express.md @@ -0,0 +1,29 @@ +# Skill: Error Handling (Express) + +## Purpose +To implement a centralized error handling mechanism in Express applications that captures synchronous and asynchronous errors, logs them properly, and sends standardized responses to the client. + +## When to Use +- When setting up the application structure. +- When refactoring to remove `try/catch` boilerplate in controllers. +- When ensuring consistent API error responses. + +## Procedure +1. **Create Custom Error Classes**: Extend `Error` (e.g., `AppError`) with properties for `statusCode` and `isOperational`. +2. **Create Error Middleware**: Define a function with signature `(err, req, res, next)`. +3. **Handle Known Errors**: + - Check if `err` is an instance of `AppError`. + - Use `err.statusCode` and `err.message`. +4. **Handle Unknown Errors**: + - Log the full error stack (for developers/logs). + - Send a generic 500 "Internal Server Error" message to the client (security). +5. **Async Wrapper**: Create a helper function (e.g., `catchAsync`) to wrap async route handlers and automatically pass errors to `next()`. +6. **Register Middleware**: Place the error handler as the *last* middleware in `app.js`. + +## Constraints +- Never expose stack traces in production responses. +- Differentiate between operational errors (invalid input) and programming errors (bugs). +- Ensure all async routes are wrapped or use Express 5+ (which handles rejected promises). + +## Expected Output +A robust error handling infrastructure including custom error classes, an error handling middleware, and an async wrapper utility. diff --git a/TRAE-Skills/backend/Express_Middleware_Creation.md b/TRAE-Skills/backend/Express_Middleware_Creation.md new file mode 100644 index 0000000000000000000000000000000000000000..028528cae94f244b4c1edd52bb87f47592e7d2ee --- /dev/null +++ b/TRAE-Skills/backend/Express_Middleware_Creation.md @@ -0,0 +1,28 @@ +# Skill: Express Middleware Creation + +## Purpose +To create reusable Express.js middleware functions for cross-cutting concerns such as logging, authentication, validation, or request preprocessing. + +## When to Use +- When logic needs to be applied to multiple routes or the entire application. +- When processing headers, cookies, or body parsing before the controller. +- When attaching custom properties to the `req` object (e.g., `req.user`). + +## Procedure +1. **Define Function Signature**: Create a function accepting `(req, res, next)`. +2. **Implement Logic**: + - Perform the specific task (e.g., check header, parse token). + - Handle errors: If an error occurs, pass it to `next(err)`. +3. **Control Flow**: + - If the request should proceed, call `next()`. + - If the request should stop (e.g., validation failure), send a response (e.g., `res.status(400).json(...)`) and do NOT call `next()`. +4. **Context Attachment**: If needed, attach data to `req` (e.g., `req.currentUser = decodedToken`). +5. **Export**: Export the function for use in routes or `app.use()`. + +## Constraints +- Always call `next()` or send a response; otherwise, the request will hang. +- Do not perform heavy blocking synchronous operations in middleware. +- Error handling middleware must have 4 arguments: `(err, req, res, next)`. + +## Expected Output +A standalone JavaScript/TypeScript file exporting a middleware function that can be mounted in an Express application. diff --git a/TRAE-Skills/backend/Feature_Flag_Implementation.md b/TRAE-Skills/backend/Feature_Flag_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..ceb1e1c49f8b58b942ec4ed1ccf73558d0dbb343 --- /dev/null +++ b/TRAE-Skills/backend/Feature_Flag_Implementation.md @@ -0,0 +1,25 @@ +# Skill: Feature Flag Implementation + +## Purpose +To enable or disable features dynamically without redeploying code, facilitating trunk-based development, A/B testing, and safe rollouts. + +## When to Use +- When merging unfinished code to the main branch. +- When rolling out a risky feature to a subset of users. +- When implementing a "kill switch" for specific functionality. + +## Procedure +1. **Choose Solution**: Simple env vars (local) or a service (LaunchDarkly, Unleash). +2. **Define Flag**: Create a boolean flag (e.g., `ENABLE_NEW_CHECKOUT`). +3. **Implement Guard**: + - In code, wrap the new logic: `if (featureFlags.isEnabled('new-checkout', user)) { ... } else { ... }`. +4. **Context Evaluation**: Pass user context (ID, email, region) to the flag evaluator to allow targeted rollouts. +5. **Cleanup**: Plan a follow-up task to remove the flag and old code once the feature is fully stable. + +## Constraints +- Do not keep flags forever; they create technical debt (complexity). +- Ensure default behavior is safe (usually "off") if the flag system fails. +- Test both "on" and "off" states. + +## Expected Output +A mechanism to toggle code paths at runtime based on configuration or user context. diff --git a/TRAE-Skills/backend/File_Storage_S3_MinIO.md b/TRAE-Skills/backend/File_Storage_S3_MinIO.md new file mode 100644 index 0000000000000000000000000000000000000000..45267f0b3aba74d534361315dfa8e42266695a39 --- /dev/null +++ b/TRAE-Skills/backend/File_Storage_S3_MinIO.md @@ -0,0 +1,46 @@ +# Skill: File Storage (S3/MinIO) + +## Purpose +To securely store, retrieve, and serve user-uploaded files (images, documents) using Object Storage, separating binary data from the database. + +## When to Use +- Storing user avatars, post images, or PDFs. +- When local disk storage is not scalable (e.g., stateless containers). + +## Procedure + +### 1. Setup +- **AWS S3**: Create Bucket, create IAM User with `PutObject`/`GetObject` permissions. +- **MinIO**: Self-hosted S3-compatible alternative (great for dev/on-prem). + +### 2. SDK Integration (AWS SDK v3) +```javascript +const { S3Client, PutObjectCommand } = require('@aws-sdk/client-s3'); +const s3 = new S3Client({ region: 'us-east-1', credentials: { ... } }); +``` + +### 3. Upload Strategy +**A. Presigned URLs (Recommended)** +Client uploads directly to S3. Server generates a temporary signature. Saves server bandwidth. +1. Server: + ```javascript + const { getSignedUrl } = require('@aws-sdk/s3-request-presigner'); + const command = new PutObjectCommand({ Bucket: 'my-bucket', Key: 'file.jpg' }); + const url = await getSignedUrl(s3, command, { expiresIn: 3600 }); + // Send url to client + ``` +2. Client: PUT binary data to that URL. + +**B. Server-Side Upload** +Client -> Server -> S3. Easier to validate content, but heavy on server. + +### 4. Security +- **Bucket Policy**: Block public access. +- **Serving Files**: Use CloudFront (CDN) with Signed URLs for private content, or public read access for public assets. + +## Constraints +- **Filename Collisions**: Always generate unique keys (e.g., `uuid-filename.ext`). +- **Validation**: Validate mime-type and file size before generating presigned URLs. + +## Expected Output +Scalable, durable file storage system with secure access controls and offloaded bandwidth usage. diff --git a/TRAE-Skills/backend/File_Upload_Handling.md b/TRAE-Skills/backend/File_Upload_Handling.md new file mode 100644 index 0000000000000000000000000000000000000000..473f0cc0fe551349f497595edc39ebf32e4ac594 --- /dev/null +++ b/TRAE-Skills/backend/File_Upload_Handling.md @@ -0,0 +1,28 @@ +# Skill: File Upload Handling (Multer/S3) + +## Purpose +To securely accept file uploads from users, validate them, and store them in a durable object storage service (like AWS S3), keeping the application server stateless. + +## When to Use +- User avatar uploads. +- Document attachments. +- Media processing pipelines. + +## Procedure +1. **Configure Middleware**: Use `multer` to handle `multipart/form-data`. +2. **Validate File**: + - Check file type (MIME type) against whitelist. + - Check file size against limit. +3. **Process Upload**: + - Stream file directly to storage (S3) using `multer-s3` (avoid saving to local disk in production). + - Generate a unique filename (UUID + extension) to prevent collisions. +4. **Save Metadata**: Store the file URL/Key and metadata in the database linked to the resource. +5. **Cleanup**: If validation fails during the process, ensure partial uploads are cleaned up. + +## Constraints +- Do not store files in the database (BLOBs) unless very small. +- Do not store user files on the application server filesystem (ephemeral). +- Scan files for viruses if possible. + +## Expected Output +An API endpoint that accepts a file, validates it, uploads it to cloud storage, and returns the file URL. diff --git a/TRAE-Skills/backend/GraphQL_Schema_Design.md b/TRAE-Skills/backend/GraphQL_Schema_Design.md new file mode 100644 index 0000000000000000000000000000000000000000..5ece3572155dbd83138a475b0f5521668030a0b3 --- /dev/null +++ b/TRAE-Skills/backend/GraphQL_Schema_Design.md @@ -0,0 +1,25 @@ +# Skill: GraphQL Schema Design + +## Purpose +To define a strongly-typed, graph-based API contract that allows clients to request exactly the data they need, minimizing over-fetching and under-fetching. + +## When to Use +- When the frontend requires flexible data fetching requirements. +- When aggregating data from multiple sources. +- When building a public API where bandwidth usage matters. + +## Procedure +1. **Define Types**: Create object types representing resources (e.g., `type User { id: ID! name: String! }`). +2. **Define Queries**: Create entry points for reading data (e.g., `user(id: ID!): User`). +3. **Define Mutations**: Create entry points for modifying data (e.g., `createUser(input: CreateUserInput!): User`). +4. **Define Resolvers**: Implement functions that fetch the actual data for each field. +5. **Handle Relationships**: Use resolvers to link types (e.g., `User` has `posts`). +6. **Schema Stitching/Federation** (Optional): If microservices, combine schemas. + +## Constraints +- Avoid N+1 query problems in resolvers (use DataLoader). +- Limit query depth to prevent DoS attacks. +- Always return nullable types for fields that might fail or be restricted. + +## Expected Output +A `.graphql` schema file or a code-first schema definition (e.g., TypeGraphQL) and corresponding resolver functions. diff --git a/TRAE-Skills/backend/GraphQL_Subscriptions_Realtime.md b/TRAE-Skills/backend/GraphQL_Subscriptions_Realtime.md new file mode 100644 index 0000000000000000000000000000000000000000..dfb6eccef276a3b62b1d384df5d75f95ecc2e9b6 --- /dev/null +++ b/TRAE-Skills/backend/GraphQL_Subscriptions_Realtime.md @@ -0,0 +1,224 @@ +# Skill: Real-time GraphQL Subscriptions + +## Purpose +To implement real-time data updates in GraphQL APIs using subscriptions over WebSockets. + +## When to Use +- When building chat applications or messaging systems +- For real-time collaboration tools (Google Docs-like apps) +- When you need live dashboards or analytics +- For real-time notifications and alerts +- When building multiplayer games + +## Procedure + +### 1. Server Setup (Apollo Server + Node.js) +Set up a GraphQL server with subscriptions. + +```javascript +import { ApolloServer } from '@apollo/server'; +import { expressMiddleware } from '@apollo/server/express4'; +import { ApolloServerPluginDrainHttpServer } from '@apollo/server/plugin/drainHttpServer'; +import { createServer } from 'http'; +import { makeExecutableSchema } from '@graphql-tools/schema'; +import { WebSocketServer } from 'ws'; +import { useServer } from 'graphql-ws/lib/use/ws'; +import express from 'express'; +import { PubSub } from 'graphql-subscriptions'; + +const pubsub = new PubSub(); +const MESSAGE_ADDED = 'MESSAGE_ADDED'; + +const typeDefs = `#graphql + type Message { + id: ID! + content: String! + user: String! + createdAt: String! + } + + type Query { + messages: [Message!]! + } + + type Mutation { + addMessage(content: String!, user: String!): Message! + } + + type Subscription { + messageAdded: Message! + } +`; + +const messages = []; + +const resolvers = { + Query: { + messages: () => messages, + }, + Mutation: { + addMessage: (_, { content, user }) => { + const message = { + id: String(messages.length + 1), + content, + user, + createdAt: new Date().toISOString(), + }; + messages.push(message); + pubsub.publish(MESSAGE_ADDED, { messageAdded: message }); + return message; + }, + }, + Subscription: { + messageAdded: { + subscribe: () => pubsub.asyncIterator([MESSAGE_ADDED]), + }, + }, +}; + +const schema = makeExecutableSchema({ typeDefs, resolvers }); + +const app = express(); +const httpServer = createServer(app); + +const wsServer = new WebSocketServer({ + server: httpServer, + path: '/graphql', +}); + +const serverCleanup = useServer({ schema }, wsServer); + +const server = new ApolloServer({ + schema, + plugins: [ + ApolloServerPluginDrainHttpServer({ httpServer }), + { + async serverWillStart() { + return { + async drainServer() { + await serverCleanup.dispose(); + }, + }; + }, + }, + ], +}); + +await server.start(); +app.use('/graphql', express.json(), expressMiddleware(server)); + +const PORT = 4000; +httpServer.listen(PORT, () => { + console.log(`Server ready at http://localhost:${PORT}/graphql`); +}); +``` + +### 2. Client Setup (Apollo Client + React) +Connect a React client to the subscription. + +```javascript +import { ApolloClient, InMemoryCache, ApolloProvider, createHttpLink, split } from '@apollo/client'; +import { GraphQLWsLink } from '@apollo/client/link/subscriptions'; +import { createClient } from 'graphql-ws'; +import { getMainDefinition } from '@apollo/client/utilities'; +import { useSubscription, gql } from '@apollo/client'; + +const httpLink = createHttpLink({ + uri: 'http://localhost:4000/graphql', +}); + +const wsLink = new GraphQLWsLink(createClient({ + url: 'ws://localhost:4000/graphql', +})); + +const splitLink = split( + ({ query }) => { + const definition = getMainDefinition(query); + return ( + definition.kind === 'OperationDefinition' && + definition.operation === 'subscription' + ); + }, + wsLink, + httpLink, +); + +const client = new ApolloClient({ + link: splitLink, + cache: new InMemoryCache(), +}); + +const MESSAGE_ADDED_SUBSCRIPTION = gql` + subscription OnMessageAdded { + messageAdded { + id + content + user + createdAt + } + } +`; + +function Chat() { + const { data } = useSubscription(MESSAGE_ADDED_SUBSCRIPTION); + + return ( +
+ {data?.messageAdded && ( +
+ {data.messageAdded.user}: {data.messageAdded.content} +
+ )} +
+ ); +} + +function App() { + return ( + + + + ); +} +``` + +### 3. Subscription Filters +Filter subscription events based on criteria. + +```javascript +const typeDefs = `#graphql + type Subscription { + messageAdded(roomId: ID!): Message! + } +`; + +const resolvers = { + Subscription: { + messageAdded: { + subscribe: (_, { roomId }) => pubsub.asyncIterator([`${MESSAGE_ADDED}_${roomId}`]), + }, + }, + Mutation: { + addMessage: (_, { content, user, roomId }) => { + const message = { + id: String(messages.length + 1), + content, + user, + roomId, + createdAt: new Date().toISOString(), + }; + messages.push(message); + pubsub.publish(`${MESSAGE_ADDED}_${roomId}`, { messageAdded: message }); + return message; + }, + }, +}; +``` + +## Best Practices +- **Authentication**: Authenticate WebSocket connections just like HTTP requests +- **Error Handling**: Handle connection errors and reconnections gracefully +- **Performance**: Use pagination for historical data, subscriptions only for new updates +- **Throttling**: Consider rate limiting subscriptions to prevent abuse +- **Scalability**: Use Redis Pub/Sub instead of in-memory PubSub for production and scaling +- **Monitoring**: Monitor WebSocket connections and subscription performance diff --git a/TRAE-Skills/backend/Health_Check_Endpoint.md b/TRAE-Skills/backend/Health_Check_Endpoint.md new file mode 100644 index 0000000000000000000000000000000000000000..8e271bfd8145fc1a36d6f80b42a0632db74593d8 --- /dev/null +++ b/TRAE-Skills/backend/Health_Check_Endpoint.md @@ -0,0 +1,31 @@ +# Skill: Health Check Endpoint Implementation + +## Purpose +To provide a standard endpoint that load balancers and orchestration systems (Kubernetes, AWS Target Groups) use to verify the application is running and ready to accept traffic. + +## When to Use +- When deploying to a container orchestration platform. +- When setting up a load balancer. +- When implementing uptime monitoring. + +## Procedure +1. **Define Liveness Probe**: + - Route: `/health/live` (or just `/health`). + - Logic: Return 200 OK immediately if the process is up. +2. **Define Readiness Probe**: + - Route: `/health/ready`. + - Logic: Check connections to critical dependencies (Database, Redis). + - If connected: Return 200 OK. + - If disconnected: Return 503 Service Unavailable. +3. **Security**: + - Ensure these endpoints are lightweight and do not expose sensitive system info. + - Optionally protect detailed info behind internal IP whitelists. +4. **Response Format**: + - JSON: `{ status: "ok", timestamp: "...", services: { db: "up", redis: "up" } }`. + +## Constraints +- Liveness check must be extremely fast and simple. +- Readiness check must not be too heavy (cache results if necessary) to avoid dDoSing the database during checks. + +## Expected Output +One or two API endpoints enabling automated systems to monitor application health and traffic readiness. diff --git a/TRAE-Skills/backend/Logger_Configuration_Winston.md b/TRAE-Skills/backend/Logger_Configuration_Winston.md new file mode 100644 index 0000000000000000000000000000000000000000..6e7c87e623f596b001369b0a10704bf82114b66c --- /dev/null +++ b/TRAE-Skills/backend/Logger_Configuration_Winston.md @@ -0,0 +1,29 @@ +# Skill: Logger Configuration (Winston) + +## Purpose +To implement a structured logging system that outputs JSON logs for machine parsing (in production) and readable text (in development), aiding in debugging and monitoring. + +## When to Use +- When replacing `console.log` statements. +- When setting up observability tools (ELK stack, Datadog, etc.). +- When needing different log levels (INFO, WARN, ERROR). + +## Procedure +1. **Install Winston**: `npm install winston`. +2. **Create Logger Instance**: + - Configure transports: `Console` (standard). + - Configure format: + - Development: `format.combine(format.colorize(), format.simple())`. + - Production: `format.combine(format.timestamp(), format.json())`. +3. **Define Levels**: Use standard syslog levels (Error, Warn, Info, HTTP, Verbose, Debug, Silly). +4. **Create Wrapper**: Export the logger instance. +5. **Integrate with Express**: Use `morgan` or a custom middleware to pipe HTTP requests into the Winston logger. +6. **Usage**: Replace `console.log` with `logger.info()`, `logger.error()`, etc. + +## Constraints +- Do not log sensitive data (passwords, tokens, PII). +- Ensure logs include context (trace ID, user ID) where possible. +- Use correct log levels (don't log errors as info). + +## Expected Output +A centralized logger module that outputs structured JSON logs in production and colored text in development. diff --git a/TRAE-Skills/backend/MFA_Implementation.md b/TRAE-Skills/backend/MFA_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..73d6eb49c2ed8b8fcddd1ff4c5a56198d6398200 --- /dev/null +++ b/TRAE-Skills/backend/MFA_Implementation.md @@ -0,0 +1,86 @@ +# Skill: MFA Implementation (2FA) + +## Purpose +To provide a second layer of security beyond passwords by requiring users to provide a one-time code generated by an authenticator app (TOTP) or sent via SMS/Email. This significantly reduces the risk of account takeovers due to stolen or weak passwords. + +## When to Use +- When protecting sensitive user accounts (banking, e-commerce, admin panels) +- As a compliance requirement for security standards (SOC2, HIPAA) +- To prevent account takeover (ATO) attacks +- When implementing "Security Checkpoints" for critical actions (e.g., changing password, withdrawing funds) + +## Procedure + +### 1. The Strategy: TOTP (Recommended) +TOTP (Time-based One-Time Password) is the gold standard for MFA. It doesn't rely on expensive SMS and works even without a cellular connection. + +### 2. Implementation: TOTP with `otplib` +Using the `otplib` and `qrcode` libraries in Node.js. + +**Installation**: +```bash +npm install otplib qrcode +``` + +**Implementation (Setup Flow)**: +```javascript +const { authenticator } = require('otplib'); +const qrcode = require('qrcode'); + +// 1. Generate a secret key for the user +const secret = authenticator.generateSecret(); + +// 2. Create a TOTP URI (to be encoded in a QR code) +const otpauth = authenticator.keyuri('user@example.com', 'MyApp', secret); + +// 3. Generate QR code for the user to scan +qrcode.toDataURL(otpauth, (err, imageUrl) => { + // Show imageUrl to user on the frontend +}); + +// 4. Save the secret key securely in the user's database record +// Important: Store it as a secret field! +await db.users.update({ where: { id: user.id }, data: { mfaSecret: secret } }); +``` + +**Implementation (Verification Flow)**: +```javascript +const { authenticator } = require('otplib'); + +async function verifyMfa(userId, tokenFromUser) { + const user = await db.users.findUnique({ where: { id: userId } }); + + // Verify the 6-digit token against the secret key + const isValid = authenticator.verify({ + token: tokenFromUser, + secret: user.mfaSecret + }); + + if (isValid) { + // MFA Success - Allow access + } else { + // MFA Failed + } +} +``` + +### 3. Handling Recovery Codes +Always provide the user with **Recovery Codes** (or "Backup Codes") when they enable MFA. These are one-time use codes that can be used if the user loses access to their authenticator app. + +```javascript +const recoveryCodes = [ + generateRandomCode(), // e.g., "ABCD-1234" + generateRandomCode(), + generateRandomCode() +]; +// Store these hashed in the database! +``` + +## Best Practices +- **Prefer TOTP over SMS**: SMS is vulnerable to "SIM swapping" attacks. Encourage users to use apps like Google Authenticator or Authy. +- **Encrypt the Secret Key**: Don't store the `mfaSecret` in plain text in your database. Use application-level encryption. +- **Provide Backup Codes**: Never let a user enable MFA without giving them backup codes. If they lose their phone, they'll be locked out forever. +- **Graceful Enrollment**: Don't force MFA on all users at once. Allow them to "remind me later" for a few days before making it mandatory. +- **Audit Logging**: Log every successful and failed MFA attempt, along with the IP address and device information. +- **Grace Periods**: Consider allowing a "Trust this device for 30 days" option to balance security and user experience. Use a persistent, secure cookie for this. +- **Revocation**: Admins must have the ability to reset or disable MFA for a user after identity verification. \ No newline at end of file diff --git a/TRAE-Skills/backend/Message_Queue_Implementation.md b/TRAE-Skills/backend/Message_Queue_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..2cf64abc95cb059dbe52542dac329699dfefcfde --- /dev/null +++ b/TRAE-Skills/backend/Message_Queue_Implementation.md @@ -0,0 +1,31 @@ +# Skill: Message Queue Implementation (RabbitMQ/SQS) + +## Purpose +To decouple heavy or long-running processes from the main HTTP request-response cycle by offloading tasks to an asynchronous background worker. + +## When to Use +- When sending emails/notifications. +- When processing video/image uploads. +- When handling high-throughput event ingestion. + +## Procedure +1. **Setup Producer**: + - Connect to the message broker (e.g., `amqplib` for RabbitMQ). + - Serialize the task payload (JSON). + - Publish the message to a specific queue or exchange. +2. **Setup Consumer (Worker)**: + - Run a separate process or service. + - Subscribe to the queue. + - Process messages one by one (or in batches). +3. **Ack/Nack Strategy**: + - If processing succeeds, acknowledge (`ack`) the message to remove it. + - If it fails, negative acknowledge (`nack`) to requeue or send to Dead Letter Queue (DLQ). +4. **Dead Letter Queue**: Configure a DLQ for messages that fail repeatedly to prevent infinite loops. + +## Constraints +- Messages must be idempotent (processing the same message twice shouldn't break things). +- Ensure the message payload size is within limits. +- Handle connection drops and retries gracefully. + +## Expected Output +A producer module for sending messages and a worker script for consuming and processing them reliably. diff --git a/TRAE-Skills/backend/Microservice_Communication.md b/TRAE-Skills/backend/Microservice_Communication.md new file mode 100644 index 0000000000000000000000000000000000000000..37cb0fa920d5ad04cf6ab4b8f6df76d40fcc58fc --- /dev/null +++ b/TRAE-Skills/backend/Microservice_Communication.md @@ -0,0 +1,27 @@ +# Skill: Microservice Communication (gRPC/HTTP) + +## Purpose +To enable reliable and efficient communication between distinct services in a distributed architecture. + +## When to Use +- Breaking a monolith into services. +- Connecting a backend to a specialized service (e.g., AI inference, Payment Gateway). + +## Procedure +1. **Choose Protocol**: + - **HTTP/REST**: Simple, ubiquitous, text-based (JSON). Good for public or loose coupling. + - **gRPC (Protobuf)**: Binary, strongly typed, high performance. Good for internal service-to-service. +2. **Define Contract**: + - REST: OpenAPI spec. + - gRPC: `.proto` file defining messages and services. +3. **Implement Client**: Create a client wrapper to make requests (handle retries, timeouts). +4. **Implement Server**: Implement the handler for the request. +5. **Service Discovery**: Mechanism to find the target service IP/Port (DNS, Consul, K8s Services). + +## Constraints +- Handle network failures (timeouts, retries with exponential backoff). +- Implement Circuit Breaker pattern to prevent cascading failures. +- Ensure backward compatibility when changing contracts. + +## Expected Output +A client-server setup allowing two distinct applications to exchange data efficiently and reliably. diff --git a/TRAE-Skills/backend/MongoDB_Aggregation.md b/TRAE-Skills/backend/MongoDB_Aggregation.md new file mode 100644 index 0000000000000000000000000000000000000000..1c4fe7c7fab5112bf0eb6092941201d4311e2cd5 --- /dev/null +++ b/TRAE-Skills/backend/MongoDB_Aggregation.md @@ -0,0 +1,76 @@ +# Skill: MongoDB Aggregation Framework + +## Purpose +To perform complex data processing, transformation, and analysis directly within MongoDB using pipelines. + +## When to Use +- When simple `find()` queries are insufficient. +- For grouping, summing, or averaging data (analytics). +- For joining collections (`$lookup`). + +## Procedure + +### 1. Structure +An aggregation pipeline is an array of stages. Documents pass through stages in order. +```javascript +db.collection.aggregate([ + { $stage1: { ... } }, + { $stage2: { ... } } +]) +``` + +### 2. Common Stages + +#### A. Filtering (`$match`) +Always place this first to reduce dataset size early. +```javascript +{ $match: { status: "completed" } } +``` + +#### B. Grouping (`$group`) +Calculate metrics. +```javascript +{ + $group: { + _id: "$customerId", // Group by customer + totalSpent: { $sum: "$amount" }, + averageOrder: { $avg: "$amount" } + } +} +``` + +#### C. Joining (`$lookup`) +Left outer join with another collection. +```javascript +{ + $lookup: { + from: "users", + localField: "userId", + foreignField: "_id", + as: "userDetails" + } +} +``` + +#### D. Projections (`$project`) +Shape the output. +```javascript +{ + $project: { + name: 1, + totalSpent: 1, + _id: 0 + } +} +``` + +### 3. Optimization +- Ensure the first `$match` stage hits an index. +- Use `$limit` and `$skip` for pagination. + +## Constraints +- **Memory Limit**: Each stage has a 100MB RAM limit. Use `{ allowDiskUse: true }` for large operations, though it is slower. +- **Complexity**: Debugging long pipelines is difficult; build them stage by stage. + +## Expected Output +Transformed and aggregated data returned efficiently without pulling all raw documents into the application layer. diff --git a/TRAE-Skills/backend/Nodejs_Streams.md b/TRAE-Skills/backend/Nodejs_Streams.md new file mode 100644 index 0000000000000000000000000000000000000000..7f0dbaea9f68111b8c1d32938022e2cfd9558815 --- /dev/null +++ b/TRAE-Skills/backend/Nodejs_Streams.md @@ -0,0 +1,66 @@ +# Skill: Node.js Streams + +## Purpose +To handle data efficiently in Node.js by processing it piece-by-piece (chunks) rather than loading everything into memory at once. + +## When to Use +- Reading/Writing large files (GBs in size). +- Processing HTTP requests/responses (e.g., video streaming, file uploads). +- Transformative pipelines (compression, encryption). + +## Procedure + +### 1. Readable Streams +Read data from a source. +```javascript +const fs = require('fs'); +const readable = fs.createReadStream('big-file.txt', { encoding: 'utf8' }); + +readable.on('data', (chunk) => { + console.log(`Received ${chunk.length} bytes`); +}); +``` + +### 2. Writable Streams +Write data to a destination. +```javascript +const writable = fs.createWriteStream('output.txt'); +writable.write('Some data\n'); +``` + +### 3. Piping (The Power of Streams) +Connect readable to writable. Handles backpressure automatically (prevents memory overflow if reading faster than writing). +```javascript +// Copy file +fs.createReadStream('input.txt').pipe(fs.createWriteStream('output.txt')); +``` + +### 4. Transform Streams +Modify data as it passes through. +```javascript +const { Transform } = require('stream'); +const gzip = require('zlib').createGzip(); + +fs.createReadStream('input.txt') + .pipe(gzip) // Compress + .pipe(fs.createWriteStream('input.txt.gz')); +``` + +### 5. Pipeline API (Modern/Safe) +Use `stream.pipeline` to handle errors correctly (automatic cleanup). +```javascript +const { pipeline } = require('stream/promises'); + +await pipeline( + fs.createReadStream('input.txt'), + gzip, + fs.createWriteStream('output.gz') +); +``` + +## Constraints +- **Error Handling**: `pipe()` does not forward errors automatically. Use `pipeline` or attach error listeners to all streams. +- **Complexity**: Writing custom Transform streams requires understanding the internal buffering mechanism. + +## Expected Output +Low memory footprint applications capable of processing datasets significantly larger than available RAM. diff --git a/TRAE-Skills/backend/OAuth2_Provider_Implementation.md b/TRAE-Skills/backend/OAuth2_Provider_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..7f8ce8b93ed48fd61fe09a342dd5f191bd9990f0 --- /dev/null +++ b/TRAE-Skills/backend/OAuth2_Provider_Implementation.md @@ -0,0 +1,59 @@ +# Skill: OAuth2 Provider Implementation + +## Purpose +To secure an API by implementing an OAuth2 authorization server, allowing third-party clients to access resources on behalf of a user. + +## When to Use +- When building a platform where other developers build apps (like "Login with Google"). +- When separating Authentication (Identity) from Authorization (Access). + +## Procedure + +### 1. Terminology +- **Resource Owner**: The User. +- **Client**: The 3rd party app. +- **Authorization Server**: Issues tokens. +- **Resource Server**: The API hosting data. + +### 2. Grant Types +Choose the right flow: +- **Authorization Code**: For server-side apps (Most common, secure). +- **PKCE**: Extension of Auth Code for mobile/SPA (Mandatory for public clients). +- **Client Credentials**: Machine-to-machine communication. + +### 3. Implementation Steps (Node.js example using `oauth2-server`) +1. **Define Model**: Implement `getClient`, `saveToken`, `getAccessToken`, `verifyScope`. + ```javascript + // model.js + module.exports = { + getClient: async (clientId, clientSecret) => { ... }, + saveToken: async (token, client, user) => { ... }, + getAccessToken: async (accessToken) => { ... }, + // ... + }; + ``` +2. **Initialize Server**: + ```javascript + const OAuth2Server = require('oauth2-server'); + const oauth = new OAuth2Server({ + model: require('./model'), + accessTokenLifetime: 3600, + allowBearerTokensInQueryString: true + }); + ``` +3. **Endpoints**: + - `POST /oauth/token`: Exchange code for token. + - `POST /oauth/authorize`: User approves access (renders login/consent page). +4. **Middleware**: Protect API routes. + ```javascript + app.get('/secure', authenticateRequest, (req, res) => { + res.json({ data: 'secret' }); + }); + ``` + +## Constraints +- **Security**: Never implement OAuth2 from scratch crypto logic. Use established libraries (`node-oauth2-server`, `panva/node-oidc-provider`). +- **HTTPS**: OAuth2 requires TLS/SSL in production to protect tokens. + +## Expected Output +A compliant OAuth2 interface issuing Access Tokens and Refresh Tokens, enabling secure delegated access. diff --git a/TRAE-Skills/backend/Pagination_Implementation.md b/TRAE-Skills/backend/Pagination_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..df539870c1f378642834687b387017fe921aaf67 --- /dev/null +++ b/TRAE-Skills/backend/Pagination_Implementation.md @@ -0,0 +1,28 @@ +# Skill: Pagination Implementation + +## Purpose +To limit the amount of data returned in a single API response, improving performance and user experience when dealing with large lists. + +## When to Use +- Listing users, products, or orders. +- Any endpoint returning an array that could grow indefinitely. + +## Procedure +1. **Choose Strategy**: + - **Offset-based** (`skip/limit`): Simple, good for page numbers. Slow on large offsets. + - **Cursor-based** (`after_id`): Faster, good for infinite scroll. No random page access. +2. **Parse Query Params**: Extract `page`, `limit`, or `cursor` from request. Set defaults (e.g., limit=20). +3. **Query Database**: + - Apply `limit` + 1 (to check if there's a next page). + - Apply sort order (crucial for cursor). +4. **Format Response**: + - Return `data` array. + - Return `meta` object: `{ total, page, hasNextPage, nextCursor }`. +5. **Validation**: Cap the maximum `limit` to prevent users requesting 10,000 rows. + +## Constraints +- Ensure the sort column is indexed. +- For cursor pagination, the sort column must be unique (or use a tiebreaker like ID). + +## Expected Output +A reusable service function or helper that wraps DB queries and returns a standardized paginated response structure. diff --git a/TRAE-Skills/backend/PostgreSQL_Indexing.md b/TRAE-Skills/backend/PostgreSQL_Indexing.md new file mode 100644 index 0000000000000000000000000000000000000000..863905f1fff4975be24ff5239c4c1489f62710cc --- /dev/null +++ b/TRAE-Skills/backend/PostgreSQL_Indexing.md @@ -0,0 +1,55 @@ +# Skill: PostgreSQL Indexing + +## Purpose +To optimize database query performance by applying appropriate indexing strategies in PostgreSQL. + +## When to Use +- When queries are slow (high execution time). +- When filtering, sorting, or joining on specific columns frequently. +- To enforce uniqueness constraints. + +## Procedure + +### 1. Analyze Queries +Use `EXPLAIN ANALYZE` to check if an index is being used. +```sql +EXPLAIN ANALYZE SELECT * FROM users WHERE email = 'test@test.com'; +-- Look for "Index Scan" vs "Seq Scan" (Sequential Scan) +``` + +### 2. Index Types + +#### A. B-Tree (Default) +Good for equality (`=`) and range (`<`, `>`, `BETWEEN`) queries. +```sql +CREATE INDEX idx_users_email ON users(email); +``` + +#### B. Hash +Good only for equality checks (`=`), smaller than B-Tree but crash-safe only in recent versions. Generally B-Tree is preferred. + +#### C. GIN (Generalized Inverted Index) +Essential for JSONB and Full-Text Search. +```sql +-- For JSONB +CREATE INDEX idx_users_metadata ON users USING GIN (metadata); +-- Query: WHERE metadata @> '{"role": "admin"}' +``` + +#### D. Partial Indexes +Index only a subset of rows to save space. +```sql +CREATE INDEX idx_active_orders ON orders(created_at) WHERE status = 'active'; +``` + +### 3. Maintenance +Indexes slow down write operations (INSERT/UPDATE). +- Remove unused indexes. +- Periodically `REINDEX` if bloat occurs (though autovacuum handles most cleanup). + +## Constraints +- **Write Performance**: Adding too many indexes degrades write speed. +- **Selectivity**: Indexes are useless for low-cardinality columns (e.g., `gender` with only 2 values) unless using partial indexes. + +## Expected Output +Significant reduction in query execution time (from seconds to milliseconds) for targeted read operations. diff --git a/TRAE-Skills/backend/Prisma_Schema_Design.md b/TRAE-Skills/backend/Prisma_Schema_Design.md new file mode 100644 index 0000000000000000000000000000000000000000..d87698ace2fe7c0d49c4cb9973b3fad206fbc911 --- /dev/null +++ b/TRAE-Skills/backend/Prisma_Schema_Design.md @@ -0,0 +1,74 @@ +# Skill: Prisma Schema Design + +## Purpose +To model application data using the Prisma Schema Language (PSL), generating a type-safe client and handling database migrations. + +## When to Use +- Modern Node.js/TypeScript backends. +- When preferring a declarative schema over SQL migrations. + +## Procedure + +### 1. Schema Definition (`schema.prisma`) +Define models, fields, and attributes. +```prisma +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") +} + +generator client { + provider = "prisma-client-js" +} + +model User { + id Int @id @default(autoincrement()) + email String @unique + posts Post[] + profile Profile? + createdAt DateTime @default(now()) +} + +model Post { + id Int @id @default(autoincrement()) + title String + content String? + published Boolean @default(false) + authorId Int + author User @relation(fields: [authorId], references: [id]) +} +``` + +### 2. Relations +- **1-1**: Optional relation on one side (`Profile?`). +- **1-n**: Array on one side (`Post[]`) and `@relation` on the other. +- **m-n**: Implicit (arrays on both sides) or Explicit (define a join model). + +### 3. Migrations +1. **Generate Migration**: Create SQL file based on schema changes. + ```bash + npx prisma migrate dev --name init_schema + ``` +2. **Generate Client**: Update the TypeScript types. + ```bash + npx prisma generate + ``` + +### 4. Usage +```typescript +const user = await prisma.user.create({ + data: { + email: 'alice@prisma.io', + posts: { + create: { title: 'Hello World' }, + }, + }, +}); +``` + +## Constraints +- **Performance**: Deeply nested writes or includes can generate complex queries. Inspect queries via logging if slow. +- **Database Features**: Some specific DB features (like Partial Indexes or specific Constraints) might need raw SQL or `@@index` configuration. + +## Expected Output +A synchronized database schema and a strongly-typed `PrismaClient` for data access. diff --git a/TRAE-Skills/backend/Real-time_Data_Processing_Kafka.md b/TRAE-Skills/backend/Real-time_Data_Processing_Kafka.md new file mode 100644 index 0000000000000000000000000000000000000000..9f849bca88ff537f768de60f02ba1c390054220d --- /dev/null +++ b/TRAE-Skills/backend/Real-time_Data_Processing_Kafka.md @@ -0,0 +1,122 @@ +# Skill: Real-time Data Processing (Kafka) + +## Purpose +To set up, configure, and use Apache Kafka for building real-time data pipelines, event streaming applications, and microservices communication. + +## When to Use +- When building real-time analytics dashboards +- For event-driven microservices architectures +- When processing high-throughput data streams (logs, metrics, user events) +- For implementing CDC (Change Data Capture) from databases +- When building real-time ETL pipelines + +## Procedure + +### 1. Kafka Producer Setup (Node.js) +Create a producer to send messages to Kafka topics. + +```javascript +const { Kafka } = require('kafkajs'); + +const kafka = new Kafka({ + clientId: 'my-app', + brokers: ['localhost:9092'] +}); + +const producer = kafka.producer(); + +async function sendMessage(topic, messages) { + await producer.connect(); + await producer.send({ + topic, + messages: messages.map(msg => ({ value: JSON.stringify(msg) })) + }); + await producer.disconnect(); +} + +// Usage +sendMessage('user-events', [ + { type: 'signup', userId: '123', email: 'user@example.com' }, + { type: 'login', userId: '123' } +]); +``` + +### 2. Kafka Consumer Setup (Node.js) +Create a consumer to process messages from topics. + +```javascript +const { Kafka } = require('kafkajs'); + +const kafka = new Kafka({ + clientId: 'analytics-consumer', + brokers: ['localhost:9092'] +}); + +const consumer = kafka.consumer({ groupId: 'analytics-group' }); + +async function consumeMessages() { + await consumer.connect(); + await consumer.subscribe({ topic: 'user-events', fromBeginning: true }); + + await consumer.run({ + eachMessage: async ({ topic, partition, message }) => { + const event = JSON.parse(message.value.toString()); + console.log('Processing event:', event); + // Process the event (e.g., update database, trigger analytics) + }, + }); +} + +consumeMessages(); +``` + +### 3. Topic Configuration +Configure topics with appropriate replication and partitions. + +```javascript +const admin = kafka.admin(); + +async function createTopic(topicName, numPartitions = 3, replicationFactor = 2) { + await admin.connect(); + await admin.createTopics({ + topics: [{ + topic: topicName, + numPartitions, + replicationFactor + }] + }); + await admin.disconnect(); +} +``` + +### 4. Error Handling & Retries +Implement robust error handling and retry logic. + +```javascript +const consumer = kafka.consumer({ + groupId: 'analytics-group', + retry: { + retries: 5, + initialRetryTime: 100, + maxRetryTime: 30000 + } +}); + +async function processWithRetry(event) { + try { + // Process the event + } catch (error) { + console.error('Error processing event:', error); + // Send to dead-letter queue for later processing + await sendMessage('dlq-user-events', [{ ...event, error: error.message }]); + } +} +``` + +## Best Practices +- **Partitioning Strategy**: Choose a good partition key to ensure even distribution of messages +- **Consumer Groups**: Use separate consumer groups for different processing needs +- **Idempotency**: Make consumers idempotent to handle duplicate messages +- **Monitoring**: Monitor lag, throughput, and error rates with tools like Prometheus +- **Dead-Letter Queues**: Implement DLQs for messages that fail processing repeatedly +- **Compaction**: Use log compaction for topics that need to retain the latest state for each key diff --git a/TRAE-Skills/backend/Redis_Data_Structures.md b/TRAE-Skills/backend/Redis_Data_Structures.md new file mode 100644 index 0000000000000000000000000000000000000000..62ec12ff167d88fdf894633eda22d2d2cfb92b56 --- /dev/null +++ b/TRAE-Skills/backend/Redis_Data_Structures.md @@ -0,0 +1,65 @@ +# Skill: Redis Data Structures + +## Purpose +To utilize Redis beyond simple key-value caching by leveraging its advanced data structures for high-performance use cases like leaderboards, queues, and real-time analytics. + +## When to Use +- **Lists/Streams**: For message queues or activity feeds. +- **Sets**: For unique collections (e.g., tags, online users). +- **Sorted Sets**: For leaderboards or priority queues. +- **Hashes**: For storing object-like data (e.g., user profiles) efficiently. +- **HyperLogLog**: For probabilistic counting (e.g., unique visitors) with minimal memory. + +## Procedure + +### 1. Connection +Using `ioredis` (Node.js): +```javascript +const Redis = require('ioredis'); +const redis = new Redis(process.env.REDIS_URL); +``` + +### 2. Common Patterns + +#### A. Leaderboards (Sorted Sets) +Use `ZADD` to add scores and `ZREVRANGE` to get top users. +```javascript +// Add user score +await redis.zadd('leaderboard', 100, 'user:1'); +await redis.zadd('leaderboard', 150, 'user:2'); + +// Get top 10 +const topUsers = await redis.zrevrange('leaderboard', 0, 9, 'WITHSCORES'); +``` + +#### B. Caching Objects (Hashes) +Use `HSET` and `HGETALL`. +```javascript +// Store user profile +await redis.hset('user:100', { + name: 'John', + email: 'john@example.com', + login_count: 5 +}); + +// Retrieve +const user = await redis.hgetall('user:100'); +``` + +#### C. Rate Limiting (Strings with Expiry) +Simple atomic counter pattern. +```javascript +const key = `rate_limit:${ip}`; +const current = await redis.incr(key); +if (current === 1) { + await redis.expire(key, 60); // 1 minute window +} +if (current > 100) throw new Error("Rate limit exceeded"); +``` + +## Constraints +- **Memory**: Redis stores everything in RAM. Use `maxmemory` policy (e.g., `allkeys-lru`) to evict old data if cache fills up. +- **Persistence**: Configure RDB or AOF if data durability is required (not just caching). + +## Expected Output +Efficient data storage and retrieval patterns tailored to specific application needs, reducing load on the primary database. diff --git a/TRAE-Skills/backend/Refactoring_Legacy_Controller.md b/TRAE-Skills/backend/Refactoring_Legacy_Controller.md new file mode 100644 index 0000000000000000000000000000000000000000..dd51abcbbd7bb554cb323f5bd4d427245e150043 --- /dev/null +++ b/TRAE-Skills/backend/Refactoring_Legacy_Controller.md @@ -0,0 +1,30 @@ +# Skill: Refactoring Legacy Controller + +## Purpose +To modernize and clean up "fat controllers" by extracting business logic into services, improving readability, and making the code testable. + +## When to Use +- When a controller file exceeds ~200 lines or handles too many responsibilities. +- When logic is duplicated across multiple controllers. +- When unit testing the controller is difficult due to mixed concerns. + +## Procedure +1. **Analyze Responsibilities**: Identify validation, business logic, data access, and response formatting mixed in the controller. +2. **Extract Service Layer**: + - Create a Service class/module for the business domain (e.g., `UserService`). + - Move business logic and DB calls to the Service. +3. **Extract Validation**: Move input validation to middleware or a separate validator helper. +4. **Refine Controller**: + - The controller should only: + 1. Parse request. + 2. Call Service method. + 3. Send response based on result. +5. **Update Tests**: Create/Update unit tests for the new Service and integration tests for the Controller. + +## Constraints +- Ensure functionality remains exactly the same (refactoring, not rewriting). +- Do not pass `req` or `res` objects into the Service layer (keep it framework-agnostic). +- Handle errors consistently (throw in Service, catch in Controller/Middleware). + +## Expected Output +A slim controller file and a corresponding service file containing the business logic, improving maintainability and testability. diff --git a/TRAE-Skills/backend/SQL_Query_Optimization.md b/TRAE-Skills/backend/SQL_Query_Optimization.md new file mode 100644 index 0000000000000000000000000000000000000000..b520999067fd8fb1c10dc9ef759385350ad1aeb7 --- /dev/null +++ b/TRAE-Skills/backend/SQL_Query_Optimization.md @@ -0,0 +1,31 @@ +# Skill: SQL Query Optimization + +## Purpose +To improve database performance by writing efficient SQL queries, using indexes correctly, and avoiding common pitfalls like N+1 problems. + +## When to Use +- When an API endpoint is slow due to database latency. +- When reviewing database access patterns in code reviews. +- When dealing with large datasets. + +## Procedure +1. **Analyze Query Plan**: Use `EXPLAIN ANALYZE` (Postgres) or equivalent to understand execution path. Look for "Seq Scan" on large tables. +2. **Add Indexes**: + - Index columns used in `WHERE`, `JOIN`, and `ORDER BY` clauses. + - Use composite indexes for multi-column queries (order matters: equality first, then range). +3. **Optimize Selects**: + - Select only necessary columns (`SELECT id, name` vs `SELECT *`). + - Avoid `SELECT *` in production code. +4. **Fix N+1 Problems**: + - Use eager loading (`JOIN` or `.withGraphFetched()` in ORMs) instead of iterating and querying in a loop. +5. **Refactor Complex Logic**: + - Move complex data manipulation to the database (aggregations, window functions) if it reduces data transfer. + - Batch inserts/updates instead of one-by-one. + +## Constraints +- Do not over-index (indexes slow down writes). +- Avoid functions on indexed columns in `WHERE` clauses (prevents index usage). +- Test performance with realistic data volumes. + +## Expected Output +Optimized SQL queries or ORM calls that execute significantly faster and consume fewer resources. diff --git a/TRAE-Skills/backend/Serverless_Streaming_Webhooks.md b/TRAE-Skills/backend/Serverless_Streaming_Webhooks.md new file mode 100644 index 0000000000000000000000000000000000000000..3985ee22d052127e78c24d029e511170b603f9b3 --- /dev/null +++ b/TRAE-Skills/backend/Serverless_Streaming_Webhooks.md @@ -0,0 +1,295 @@ +# Skill: Serverless Streaming & Webhooks at Scale + +## Purpose +To build high-throughput, real-time streaming APIs and webhook systems using serverless technologies. + +## When to Use +- For real-time LLM streaming (OpenAI, Anthropic) +- When building webhook systems for SaaS integrations +- For high-throughput event processing +- When you need serverless SSE (Server-Sent Events) +- For building async task queues with webhook callbacks + +## Procedure + +### 1. LLM Streaming (Next.js Edge Function) +Stream LLM responses from a serverless function. + +```typescript +// app/api/chat/stream/route.ts +import OpenAI from 'openai'; +import { OpenAIStream, StreamingTextResponse } from 'ai'; + +export const runtime = 'edge'; + +const openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, +}); + +export async function POST(req: Request) { + const { messages } = await req.json(); + + const response = await openai.chat.completions.create({ + model: 'gpt-4o', + stream: true, + messages, + }); + + const stream = OpenAIStream(response, { + onStart: async () => { + console.log('Stream started'); + }, + onCompletion: async (completion) => { + console.log('Stream completed:', completion); + // Save to database + }, + }); + + return new StreamingTextResponse(stream); +} +``` + +### 2. Client-Side Streaming +Consume the stream in React. + +```tsx +// app/chat/page.tsx +'use client'; + +import { useChat } from 'ai/react'; + +export default function ChatPage() { + const { messages, input, handleInputChange, handleSubmit } = useChat({ + api: '/api/chat/stream', + onResponse: (response) => { + console.log('Response received:', response); + }, + onFinish: (message) => { + console.log('Finished:', message); + }, + }); + + return ( +
+
+ {messages.map((m) => ( +
+ {m.role}: {m.content} +
+ ))} +
+ +
+ + +
+
+ ); +} +``` + +### 3. Webhook System (AWS Lambda + SQS) +Build a scalable webhook system with retries. + +```typescript +// serverless.yml +service: webhook-service + +provider: + name: aws + runtime: nodejs20.x + iam: + role: + statements: + - Effect: Allow + Action: + - sqs:SendMessage + - sqs:ReceiveMessage + - sqs:DeleteMessage + Resource: !GetAtt WebhookQueue.Arn + +functions: + sendWebhook: + handler: handler.sendWebhook + events: + - sqs: + arn: !GetAtt WebhookQueue.Arn + batchSize: 1 + +resources: + Resources: + WebhookQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: webhook-queue + VisibilityTimeout: 60 + RedrivePolicy: + deadLetterTargetArn: !GetAtt WebhookDLQ.Arn + maxReceiveCount: 5 + WebhookDLQ: + Type: AWS::SQS::Queue + Properties: + QueueName: webhook-dlq +``` + +```typescript +// handler.ts +import { SQSClient, SendMessageCommand } from '@aws-sdk/client-sqs'; +import axios from 'axios'; + +const sqs = new SQSClient({ region: process.env.AWS_REGION }); +const QUEUE_URL = process.env.WEBHOOK_QUEUE_URL!; + +// Send webhook to queue (triggered by your app) +export async function queueWebhook(url: string, payload: any) { + const command = new SendMessageCommand({ + QueueUrl: QUEUE_URL, + MessageBody: JSON.stringify({ + url, + payload, + attempt: 1, + maxAttempts: 5, + createdAt: new Date().toISOString() + }), + }); + + await sqs.send(command); + return { success: true }; +} + +// Process webhook from queue +export async function sendWebhook(event: any) { + for (const record of event.Records) { + const { url, payload, attempt, maxAttempts } = JSON.parse(record.body); + + try { + await axios.post(url, payload, { + timeout: 5000, + headers: { + 'Content-Type': 'application/json', + 'X-Webhook-Signature': generateSignature(payload), + }, + }); + + console.log(`Webhook sent to ${url} (attempt ${attempt})`); + } catch (error) { + console.error(`Webhook failed to ${url} (attempt ${attempt}):`, error); + + if (attempt < maxAttempts) { + // Requeue with backoff + await queueWebhook(url, payload); + } else { + // Send to DLQ or alert + await sendToDLQ(record.body, error); + } + } + } +} + +function generateSignature(payload: any) { + const hmac = crypto.createHmac('sha256', process.env.WEBHOOK_SECRET!); + return hmac.update(JSON.stringify(payload)).digest('hex'); +} +``` + +### 4. Server-Sent Events (SSE) +Build real-time updates with SSE. + +```typescript +// app/api/sse/route.ts +import { NextRequest } from 'next/server'; + +export const runtime = 'edge'; + +export async function GET(request: NextRequest) { + const encoder = new TextEncoder(); + + const stream = new ReadableStream({ + async start(controller) { + // Send initial message + controller.enqueue(encoder.encode(`data: ${JSON.stringify({ type: 'connected' })}\n\n`)); + + // Send periodic updates + const interval = setInterval(() => { + const event = { + type: 'update', + timestamp: new Date().toISOString(), + data: Math.random(), + }; + controller.enqueue(encoder.encode(`data: ${JSON.stringify(event)}\n\n`)); + }, 2000); + + // Cleanup + request.signal.addEventListener('abort', () => { + clearInterval(interval); + controller.close(); + }); + }, + }); + + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + }, + }); +} +``` + +```tsx +// app/sse/page.tsx +'use client'; + +import { useEffect, useState } from 'react'; + +export default function SSEPage() { + const [events, setEvents] = useState([]); + + useEffect(() => { + const eventSource = new EventSource('/api/sse'); + + eventSource.onmessage = (event) => { + const data = JSON.parse(event.data); + setEvents(prev => [...prev, data]); + }; + + eventSource.onerror = (error) => { + console.error('SSE error:', error); + eventSource.close(); + }; + + return () => eventSource.close(); + }, []); + + return ( +
+

SSE Updates

+
+ {events.map((event, i) => ( +
+ {JSON.stringify(event)} +
+ ))} +
+
+ ); +} +``` + +## Best Practices +- **Retries with Backoff**: Implement exponential backoff for webhooks +- **Idempotency**: Add idempotency keys to avoid duplicate processing +- **Signatures**: Sign webhooks for security +- **Dead Letter Queues**: Use DLQs for failed webhooks +- **Timeouts**: Set reasonable timeouts for streaming +- **Monitoring**: Monitor webhook delivery rates +- **Rate Limiting**: Respect third-party rate limits +- **Error Alerting**: Alert on DLQ messages diff --git a/TRAE-Skills/backend/Session_Management_Best_Practices.md b/TRAE-Skills/backend/Session_Management_Best_Practices.md new file mode 100644 index 0000000000000000000000000000000000000000..54880bb1460ba34a9211e235a452dda80fa6bd2e --- /dev/null +++ b/TRAE-Skills/backend/Session_Management_Best_Practices.md @@ -0,0 +1,56 @@ +# Skill: Session Management Best Practices + +## Purpose +To securely manage user sessions, maintaining state across stateless HTTP requests while preventing hijacking and XSS/CSRF attacks. + +## When to Use +- Any web application requiring login. +- Choosing between JWT and Server-side Sessions. + +## Procedure + +### 1. Choice: JWT vs Session Cookies +- **Session Cookies (Stateful)**: Better for security (revocation is easy). ID stored in cookie, data in Redis/DB. +- **JWT (Stateless)**: Better for scalability/microservices. Data signed in token. Harder to revoke. + +### 2. Secure Cookie Configuration +Regardless of the method, store the token/ID in a **HttpOnly** cookie. +```javascript +res.cookie('session_id', token, { + httpOnly: true, // Prevents JS access (XSS protection) + secure: process.env.NODE_ENV === 'production', // HTTPS only + sameSite: 'strict', // CSRF protection + maxAge: 24 * 60 * 60 * 1000 // 1 day +}); +``` + +### 3. Session Store (for Stateful) +Don't use memory store in production. Use Redis. +```javascript +const session = require('express-session'); +const RedisStore = require('connect-redis').default; + +app.use(session({ + store: new RedisStore({ client: redisClient }), + secret: process.env.SESSION_SECRET, + resave: false, + saveUninitialized: false, + cookie: { secure: true, httpOnly: true } +})); +``` + +### 4. Rotation & Expiry +- **Regenerate Session ID** on login to prevent Session Fixation attacks. + ```javascript + req.session.regenerate((err) => { + // save new session + }); + ``` +- Implement **absolute timeouts** (e.g., force re-login after 7 days) and **idle timeouts** (logout after 30m inactivity). + +## Constraints +- **Mobile Apps**: Often prefer JWT in Authorization header (`Bearer token`) rather than cookies, as they don't have a browser environment. +- **Size**: Cookies have a 4KB limit. Don't store large data in them. + +## Expected Output +A secure authentication state mechanism resilient to common web attacks. diff --git a/TRAE-Skills/backend/Soft_Delete_Implementation.md b/TRAE-Skills/backend/Soft_Delete_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..bcae440ccaf232169230fa0e34f2452717c8c259 --- /dev/null +++ b/TRAE-Skills/backend/Soft_Delete_Implementation.md @@ -0,0 +1,24 @@ +# Skill: Soft Delete Implementation + +## Purpose +To mark records as deleted without physically removing them from the database, allowing for data recovery and audit trails. + +## When to Use +- When data is valuable or legally required to be retained. +- When "deleting" a user account or order. + +## Procedure +1. **Schema Change**: Add a `deletedAt` (timestamp) column to the table. Nullable. +2. **Modify Delete Logic**: + - Instead of `DELETE FROM table`, perform `UPDATE table SET deletedAt = NOW()`. +3. **Modify Read Logic**: + - Filter queries globally or per-query: `WHERE deletedAt IS NULL`. +4. **Cascade**: Decide how to handle related records (soft delete them too, or just hide them via the parent's status). +5. **Restore**: Implement a `restore` method that sets `deletedAt = NULL`. + +## Constraints +- Unique indexes might conflict (e.g., email must be unique, but a soft-deleted user has the email). Solution: Include `deletedAt` in the unique index or use partial indexes. +- Remember to filter soft-deleted records in *every* select query. + +## Expected Output +A data access layer that transparently handles soft deletion and filtering, ensuring deleted items don't appear in normal results. diff --git a/TRAE-Skills/backend/Swagger_Documentation.md b/TRAE-Skills/backend/Swagger_Documentation.md new file mode 100644 index 0000000000000000000000000000000000000000..f6c00d5beafc093af04349d1cf0f7fcb2341dcad --- /dev/null +++ b/TRAE-Skills/backend/Swagger_Documentation.md @@ -0,0 +1,29 @@ +# Skill: Swagger Documentation Generation + +## Purpose +To automatically generate interactive API documentation that stays up-to-date with the codebase, aiding frontend developers and third-party integrators. + +## When to Use +- When exposing APIs to other teams or the public. +- When strictly defining API contracts. +- When standardizing API development. + +## Procedure +1. **Install Tools**: `swagger-jsdoc` and `swagger-ui-express`. +2. **Configure Definition**: Create a `swaggerDef.js` or equivalent config with API meta-info (title, version, server URL). +3. **Annotate Code**: + - Add JSDoc/YAML comments above route handlers describing: + - Path and Method. + - Parameters and Request Body. + - Response Codes and Schemas. + - Alternatively, use a library that infers docs from validation schemas (e.g., `zod-to-openapi`). +4. **Mount UI**: Add a route (e.g., `/api-docs`) to serve the Swagger UI using the generated spec. +5. **Secure Docs**: Protect the documentation route if the API is private. + +## Constraints +- Documentation must match the actual implementation. +- Keep schemas reusable (use `$ref` definitions). +- Do not expose sensitive internal endpoints in public documentation. + +## Expected Output +A live `/api-docs` endpoint displaying interactive Swagger UI, generated from code annotations or schemas. diff --git a/TRAE-Skills/backend/Transaction_Management.md b/TRAE-Skills/backend/Transaction_Management.md new file mode 100644 index 0000000000000000000000000000000000000000..ee504a06fef4a482859ddd182f9792e3958eab45 --- /dev/null +++ b/TRAE-Skills/backend/Transaction_Management.md @@ -0,0 +1,23 @@ +# Skill: Transaction Management + +## Purpose +To ensure data integrity by grouping multiple database operations into a single atomic unit. Either all operations succeed, or none do. + +## When to Use +- Transferring money (debit A, credit B). +- Creating an order (create order, create items, update inventory). +- Any multi-step write operation. + +## Procedure +1. **Start Transaction**: Begin the transaction via the ORM/DB client. +2. **Execute Operations**: Run the sequence of DB commands, passing the transaction object/context to each. +3. **Commit**: If all steps succeed, commit the transaction to persist changes. +4. **Rollback**: If any step throws an error, catch it and rollback the transaction to revert all changes. +5. **Release**: Release the connection back to the pool. + +## Constraints +- Keep transactions short to avoid locking tables for too long. +- Do not perform external API calls inside a transaction (if the API succeeds but DB fails, you can't rollback the API). + +## Expected Output +A robust function wrapper or pattern that handles the begin/commit/rollback lifecycle for complex operations. diff --git a/TRAE-Skills/backend/TypeORM_Relations.md b/TRAE-Skills/backend/TypeORM_Relations.md new file mode 100644 index 0000000000000000000000000000000000000000..8c6e29fc991bd951f6cd419fe38925f809b21ec3 --- /dev/null +++ b/TRAE-Skills/backend/TypeORM_Relations.md @@ -0,0 +1,72 @@ +# Skill: TypeORM Relations + +## Purpose +To define and manage relationships (One-to-One, One-to-Many, Many-to-Many) between entities in TypeScript applications using TypeORM. + +## When to Use +- Building a relational backend with Node.js/TypeScript. +- Need to load related data automatically (Eager) or on demand (Lazy). + +## Procedure + +### 1. One-to-Many / Many-to-One +Example: User has many Photos. + +**User Entity**: +```typescript +@Entity() +export class User { + @PrimaryGeneratedColumn() + id: number; + + @OneToMany(() => Photo, (photo) => photo.user) + photos: Photo[]; +} +``` + +**Photo Entity**: +```typescript +@Entity() +export class Photo { + @PrimaryGeneratedColumn() + id: number; + + @ManyToOne(() => User, (user) => user.photos) + user: User; +} +``` + +### 2. Many-to-Many +Example: Category has many Questions. Requires a join table (handled automatically or manually). +```typescript +@Entity() +export class Category { + @PrimaryGeneratedColumn() + id: number; + + @ManyToMany(() => Question, (question) => question.categories) + @JoinTable() // Only on one side (the "owner") + questions: Question[]; +} +``` + +### 3. Loading Relations +- **Find Options**: + ```typescript + const users = await userRepository.find({ + relations: { photos: true } + }); + ``` +- **QueryBuilder**: + ```typescript + const users = await userRepository.createQueryBuilder("user") + .leftJoinAndSelect("user.photos", "photo") + .getMany(); + ``` + +## Constraints +- **N+1 Problem**: Avoid looping over entities and fetching relations one by one. Use `relations` or `leftJoinAndSelect` to fetch in a single query. +- **Circular References**: Be careful when serializing entities to JSON; use `class-transformer` to exclude circular fields. + +## Expected Output +Correctly linked database tables and the ability to fetch related data graphs programmatically. diff --git a/TRAE-Skills/backend/Type_Safe_APIs_tRPC.md b/TRAE-Skills/backend/Type_Safe_APIs_tRPC.md new file mode 100644 index 0000000000000000000000000000000000000000..aec76e8798810e95aa807b78e7f2d2756b4d5f92 --- /dev/null +++ b/TRAE-Skills/backend/Type_Safe_APIs_tRPC.md @@ -0,0 +1,303 @@ +# Skill: Type-Safe APIs with tRPC + +## Purpose +To build end-to-end type-safe APIs without schemas or code generation, sharing types between frontend and backend. + +## When to Use +- When building full-stack TypeScript applications +- For faster development with type safety +- When you want to avoid API schema duplication +- For Next.js apps (tRPC integrates perfectly) +- When you want auto-completion for API calls + +## Procedure + +### 1. Basic tRPC Setup (Next.js App Router) +Set up tRPC with Next.js App Router. + +```typescript +// server/api/trpc/[trpc]/route.ts +import { initTRPC } from '@trpc/server'; +import { fetchRequestHandler } from '@trpc/server/adapters/fetch'; + +const t = initTRPC.create(); + +// Define procedures +const publicProcedure = t.procedure; +const router = t.router; + +// Create app router +const appRouter = router({ + hello: publicProcedure + .input((val: unknown) => { + if (typeof val === 'string') return val; + throw new Error('Invalid input'); + }) + .query(({ input }) => { + return { + message: `Hello, ${input}!`, + timestamp: new Date().toISOString() + }; + }), + addUser: publicProcedure + .input((val: unknown) => { + if (typeof val !== 'object' || val === null) throw new Error('Invalid input'); + const { name, email } = val as { name: string; email: string }; + if (typeof name !== 'string' || typeof email !== 'string') throw new Error('Invalid input'); + return { name, email }; + }) + .mutation(({ input }) => { + return { + id: Math.random().toString(36).substr(2, 9), + name: input.name, + email: input.email, + createdAt: new Date().toISOString() + }; + }) +}); + +// Export router type +export type AppRouter = typeof appRouter; + +// Handler +function handler(req: Request) { + return fetchRequestHandler({ + endpoint: '/api/trpc', + req, + router: appRouter, + createContext: () => ({}), + }); +} + +export { handler as GET, handler as POST }; +``` + +### 2. Client Setup +Set up the tRPC client. + +```typescript +// lib/trpc.ts +import { createTRPCReact } from '@trpc/react-query'; +import type { AppRouter } from '@/server/api/trpc/[trpc]/route'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { httpBatchLink } from '@trpc/client'; + +export const trpc = createTRPCReact(); + +export function TRPCProvider({ children }: { children: React.ReactNode }) { + const [queryClient] = useState(() => new QueryClient()); + const [trpcClient] = useState(() => + trpc.createClient({ + links: [ + httpBatchLink({ + url: '/api/trpc', + }), + ], + }) + ); + + return ( + + {children} + + ); +} +``` + +### 3. Use in Components +Call tRPC procedures in React components. + +```tsx +// app/page.tsx +'use client'; + +import { trpc } from '@/lib/trpc'; + +export default function Home() { + // Query procedure + const { data: helloData, isLoading: helloLoading } = trpc.hello.useQuery('World'); + + // Mutation procedure + const addUserMutation = trpc.addUser.useMutation(); + const [name, setName] = useState(''); + const [email, setEmail] = useState(''); + + const handleAddUser = async () => { + const result = await addUserMutation.mutateAsync({ name, email }); + console.log('User added:', result); + setName(''); + setEmail(''); + }; + + return ( +
+

tRPC Demo

+ +
+

Hello Query

+ {helloLoading ?

Loading...

:

{helloData?.message}

} +
+ +
+

Add User Mutation

+ setName(e.target.value)} + className="border p-2 mr-2" + /> + setEmail(e.target.value)} + className="border p-2 mr-2" + /> + + {addUserMutation.data && ( +

+ User added: {addUserMutation.data.name} ({addUserMutation.data.id}) +

+ )} +
+
+ ); +} +``` + +### 4. Middleware & Context +Add authentication and context. + +```typescript +// server/api/trpc/[trpc]/route.ts +import { initTRPC, TRPCError } from '@trpc/server'; +import { getServerSession } from 'next-auth'; + +const t = initTRPC.context().create(); + +// Context +const createContext = async (opts: any) => { + const session = await getServerSession(); + return { + session, + user: session?.user, + }; +}; + +// Middleware +const isAuthenticated = t.middleware(({ ctx, next }) => { + if (!ctx.user) { + throw new TRPCError({ code: 'UNAUTHORIZED' }); + } + return next({ + ctx: { + ...ctx, + user: ctx.user, + }, + }); +}); + +const protectedProcedure = t.procedure.use(isAuthenticated); + +// App router +const appRouter = router({ + // Public procedures + hello: publicProcedure.query(() => ({ message: 'Hello!' })), + + // Protected procedures + getProfile: protectedProcedure + .query(({ ctx }) => { + return { + id: ctx.user.id, + name: ctx.user.name, + email: ctx.user.email, + }; + }), + + // With Zod validation + createPost: protectedProcedure + .input(z.object({ + title: z.string().min(1).max(100), + content: z.string().min(1), + tags: z.array(z.string()).optional(), + })) + .mutation(({ ctx, input }) => { + return { + id: Math.random().toString(36).substr(2, 9), + title: input.title, + content: input.content, + tags: input.tags || [], + authorId: ctx.user.id, + createdAt: new Date().toISOString() + }; + }) +}); +``` + +### 5. Subscriptions (WebSockets) +Real-time subscriptions with tRPC. + +```typescript +// Server +import { observable } from '@trpc/server/observable'; + +const appRouter = router({ + time: publicProcedure.subscription(() => { + return observable((emit) => { + const interval = setInterval(() => { + emit.next(new Date().toISOString()); + }, 1000); + + return () => clearInterval(interval); + }); + }), + chat: protectedProcedure + .input(z.object({ roomId: z.string() })) + .subscription(({ ctx, input }) => { + return observable((emit) => { + // Subscribe to chat room + const unsubscribe = subscribeToRoom(input.roomId, (message) => { + emit.next(message); + }); + return unsubscribe; + }); + }) +}); + +// Client +function ChatRoom({ roomId }: { roomId: string }) { + const [messages, setMessages] = useState([]); + + const chatSubscription = trpc.chat.useSubscription({ roomId }, { + onData: (message) => { + setMessages(prev => [...prev, message]); + }, + }); + + return ( +
+ {chatSubscription.isLoading &&

Connecting...

} + {messages.map((msg, i) => ( +

{msg}

+ ))} +
+ ); +} +``` + +## Best Practices +- **Zod Validation**: Always use Zod for input validation +- **Procedure Organization**: Group related procedures in routers +- **Middleware**: Use middleware for auth, logging, rate limiting +- **Error Handling**: Throw TRPCError with appropriate codes +- **React Query**: Leverage React Query for caching and invalidation +- **Batching**: Use httpBatchLink to batch requests +- **Type Safety**: Never use `any`, rely on inferred types +- **Inference**: Let tRPC infer types automatically diff --git a/TRAE-Skills/backend/WebSocket_Implementation.md b/TRAE-Skills/backend/WebSocket_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..de83aa875f5d0683affdd885fe5000072adceb29 --- /dev/null +++ b/TRAE-Skills/backend/WebSocket_Implementation.md @@ -0,0 +1,27 @@ +# Skill: WebSocket Implementation (Socket.io) + +## Purpose +To enable real-time, bi-directional event-based communication between the client and server. + +## When to Use +- Chat applications. +- Live notifications/alerts. +- Real-time dashboards or collaborative editing. + +## Procedure +1. **Initialize Server**: Attach Socket.io to the HTTP server instance. +2. **Handle Connection**: Listen for `connection` events. +3. **Authentication**: Validate the handshake (e.g., verify JWT in query params or headers) before allowing connection. +4. **Define Events**: + - `emit` to send data to client(s). + - `on` to listen for data from client. +5. **Rooms/Namespaces**: Use rooms to group sockets (e.g., `socket.join('room_123')`) for targeted broadcasting. +6. **Handle Disconnect**: Clean up resources when a user leaves. + +## Constraints +- WebSockets are stateful; scaling requires a Redis adapter to sync events across multiple server instances. +- Handle reconnection logic on the client side. +- Be mindful of open connection limits. + +## Expected Output +A WebSocket server module handling connection, auth, and event routing for real-time features. diff --git a/TRAE-Skills/backend/WebSocket_Scalability_SocketIO.md b/TRAE-Skills/backend/WebSocket_Scalability_SocketIO.md new file mode 100644 index 0000000000000000000000000000000000000000..fe635c3121f061e1eb368959856ee196922909f0 --- /dev/null +++ b/TRAE-Skills/backend/WebSocket_Scalability_SocketIO.md @@ -0,0 +1,252 @@ +# Skill: Scalable WebSocket Applications with Socket.IO + +## Purpose +To build and scale real-time WebSocket applications using Socket.IO with Redis adapter for horizontal scaling. + +## When to Use +- When building chat applications with many concurrent users +- For real-time collaboration tools +- When you need to scale WebSocket servers horizontally +- For live dashboards and real-time analytics +- When building multiplayer games + +## Procedure + +### 1. Basic Socket.IO Server +Set up a simple Socket.IO server. + +```javascript +import express from 'express'; +import http from 'http'; +import { Server } from 'socket.io'; + +const app = express(); +const server = http.createServer(app); +const io = new Server(server, { + cors: { + origin: "https://your-frontend.com", + methods: ["GET", "POST"] + } +}); + +io.on('connection', (socket) => { + console.log('User connected:', socket.id); + + socket.on('join-room', (roomId) => { + socket.join(roomId); + console.log('User', socket.id, 'joined room', roomId); + }); + + socket.on('send-message', (data) => { + io.to(data.roomId).emit('new-message', { + id: Date.now(), + content: data.content, + user: data.user, + timestamp: new Date().toISOString() + }); + }); + + socket.on('disconnect', () => { + console.log('User disconnected:', socket.id); + }); +}); + +const PORT = process.env.PORT || 3000; +server.listen(PORT, () => { + console.log(`Server running on port ${PORT}`); +}); +``` + +### 2. Socket.IO Client +Connect from the frontend. + +```javascript +import { io } from 'socket.io-client'; + +const socket = io('https://your-server.com'); + +socket.on('connect', () => { + console.log('Connected with ID:', socket.id); +}); + +socket.on('new-message', (message) => { + console.log('New message:', message); + // Update UI with new message +}); + +// Join a room +socket.emit('join-room', 'room-123'); + +// Send a message +socket.emit('send-message', { + roomId: 'room-123', + content: 'Hello, everyone!', + user: 'John Doe' +}); +``` + +### 3. Scale with Redis Adapter +Enable horizontal scaling with Redis. + +```javascript +import express from 'express'; +import http from 'http'; +import { Server } from 'socket.io'; +import { createAdapter } from '@socket.io/redis-adapter'; +import { createClient } from 'redis'; + +const app = express(); +const server = http.createServer(app); + +const io = new Server(server, { + cors: { + origin: "https://your-frontend.com", + methods: ["GET", "POST"] + } +}); + +// Set up Redis adapter +const pubClient = createClient({ host: 'localhost', port: 6379 }); +const subClient = pubClient.duplicate(); + +Promise.all([pubClient.connect(), subClient.connect()]).then(() => { + io.adapter(createAdapter(pubClient, subClient)); + console.log('Redis adapter enabled'); +}); + +io.on('connection', (socket) => { + console.log('User connected:', socket.id); + + socket.on('join-room', (roomId) => { + socket.join(roomId); + // Notify everyone in the room + io.to(roomId).emit('user-joined', socket.id); + }); + + socket.on('send-message', (data) => { + io.to(data.roomId).emit('new-message', { + id: Date.now(), + content: data.content, + user: data.user, + timestamp: new Date().toISOString() + }); + }); + + socket.on('disconnect', () => { + console.log('User disconnected:', socket.id); + }); +}); + +const PORT = process.env.PORT || 3000; +server.listen(PORT, () => { + console.log(`Server running on port ${PORT}`); +}); +``` + +### 4. Load Balancing with Nginx +Configure Nginx for load balancing. + +```nginx +http { + upstream socketio_servers { + least_conn; + server localhost:3001; + server localhost:3002; + server localhost:3003; + } + + server { + listen 80; + server_name your-domain.com; + + location /socket.io/ { + proxy_pass http://socketio_servers/socket.io/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket timeout settings + proxy_connect_timeout 7d; + proxy_send_timeout 7d; + proxy_read_timeout 7d; + } + } +} +``` + +### 5. Authentication +Add JWT authentication to Socket.IO. + +```javascript +import jwt from 'jsonwebtoken'; + +io.use((socket, next) => { + const token = socket.handshake.auth.token; + if (!token) { + return next(new Error('Authentication required')); + } + + try { + const decoded = jwt.verify(token, 'your-secret-key'); + socket.user = decoded; + next(); + } catch (error) { + return next(new Error('Invalid token')); + } +}); + +io.on('connection', (socket) => { + console.log('Authenticated user:', socket.user.id); + + socket.on('send-message', (data) => { + io.to(data.roomId).emit('new-message', { + id: Date.now(), + content: data.content, + userId: socket.user.id, + username: socket.user.name, + timestamp: new Date().toISOString() + }); + }); +}); +``` + +### 6. Presence Tracking +Track online users. + +```javascript +const onlineUsers = new Map(); + +io.on('connection', (socket) => { + const userId = socket.user.id; + onlineUsers.set(userId, { + id: userId, + name: socket.user.name, + socketId: socket.id + }); + + // Notify everyone that this user came online + io.emit('user-online', { userId, name: socket.user.name }); + + // Send list of online users to new user + socket.emit('online-users', Array.from(onlineUsers.values())); + + socket.on('disconnect', () => { + onlineUsers.delete(userId); + io.emit('user-offline', userId); + }); +}); +``` + +## Best Practices +- **Always Use Adapter**: Use Redis (or other) adapter for scaling +- **Connection State Recovery**: Enable connection state recovery +- **Heartbeats**: Configure appropriate heartbeat intervals +- **Error Handling**: Handle errors gracefully +- **Rate Limiting**: Implement rate limiting to prevent abuse +- **Message Acknowledgements**: Use acknowledgements for important messages +- **Rooms**: Use rooms for efficient message broadcasting +- **Monitoring**: Monitor connection count and message rates diff --git a/TRAE-Skills/backend/Webhooks_Implementation.md b/TRAE-Skills/backend/Webhooks_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..12c6d7f20540d4324e46c6dff1db7bd89779f4e6 --- /dev/null +++ b/TRAE-Skills/backend/Webhooks_Implementation.md @@ -0,0 +1,40 @@ +# Skill: Webhooks Implementation + +## Purpose +To implement a system that pushes real-time event notifications to external subscribers via HTTP POST requests. + +## When to Use +- Notifying 3rd party integrations of state changes (e.g., "Payment Successful", "Order Shipped"). +- Reducing polling from clients. + +## Procedure + +### 1. Subscription Management +Create a database table to store subscriber URLs and the events they are interested in. +- `id`, `user_id`, `target_url`, `event_types` (array), `secret_key` (for signing). + +### 2. Event Triggering +When an event occurs in the system: +1. Construct the payload: `{ id: "evt_123", type: "order.created", data: { ... } }`. +2. Find active subscriptions for this event type. +3. Enqueue a background job (using BullMQ/RabbitMQ) to send the webhook. **Do not send synchronously** to avoid blocking the request. + +### 3. Delivery (The Worker) +1. **Signature**: Generate an HMAC signature of the payload using the subscriber's secret key. + ```javascript + const signature = crypto.createHmac('sha256', secret).update(JSON.stringify(payload)).digest('hex'); + ``` +2. **Request**: Send POST request with signature header (e.g., `X-Signature`). +3. **Retry Logic**: Implement exponential backoff if the subscriber returns 5xx or times out. Disable subscription after N consecutive failures. + +### 4. Security +- **Signing**: Allows receivers to verify the request came from you. +- **Timeouts**: Set short timeouts (e.g., 5s) to prevent stuck workers. +- **SSRF Prevention**: Validate `target_url` to ensure it doesn't point to internal network addresses (localhost, 192.168.x.x). + +## Constraints +- **Ordering**: Delivery order is not guaranteed in distributed systems. Include a timestamp in the payload. +- **Idempotency**: Receivers should handle duplicate webhooks gracefully (using event ID). + +## Expected Output +A robust event delivery system allowing external systems to react to internal changes in near real-time. diff --git a/TRAE-Skills/backend/gRPC_Service_Implementation.md b/TRAE-Skills/backend/gRPC_Service_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..2f32f820b1aa06102f576bcb67d6bdf2a5800b62 --- /dev/null +++ b/TRAE-Skills/backend/gRPC_Service_Implementation.md @@ -0,0 +1,70 @@ +# Skill: gRPC Service Implementation + +## Purpose +To build high-performance, low-latency inter-service communication using Protocol Buffers and HTTP/2. + +## When to Use +- Microservices architectures. +- Internal APIs where low latency is critical. +- Polyglot environments (services in Go, Java, Python, Node). + +## Procedure + +### 1. Define Protocol Buffer (.proto) +Define the service contract and message structures. +```protobuf +syntax = "proto3"; + +package greeter; + +service Greeter { + rpc SayHello (HelloRequest) returns (HelloReply) {} +} + +message HelloRequest { + string name = 1; +} + +message HelloReply { + string message = 1; +} +``` + +### 2. Server Implementation (Node.js) +1. **Load Proto**: + ```javascript + const grpc = require('@grpc/grpc-js'); + const protoLoader = require('@grpc/proto-loader'); + const packageDefinition = protoLoader.loadSync('greeter.proto'); + const greeterProto = grpc.loadPackageDefinition(packageDefinition).greeter; + ``` +2. **Implement Logic**: + ```javascript + function sayHello(call, callback) { + callback(null, { message: 'Hello ' + call.request.name }); + } + ``` +3. **Start Server**: + ```javascript + const server = new grpc.Server(); + server.addService(greeterProto.Greeter.service, { sayHello: sayHello }); + server.bindAsync('0.0.0.0:50051', grpc.ServerCredentials.createInsecure(), () => { + server.start(); + }); + ``` + +### 3. Client Implementation +```javascript +const client = new greeterProto.Greeter('localhost:50051', grpc.credentials.createInsecure()); + +client.sayHello({ name: 'World' }, (err, response) => { + console.log('Greeting:', response.message); +}); +``` + +## Constraints +- **Browser Support**: Browsers do not support gRPC natively. Use gRPC-Web proxy if frontend access is needed. +- **Debugging**: Binary format is not human-readable. Use tools like BloomRPC or Postman (gRPC support) for testing. + +## Expected Output +Strongly-typed, efficient communication channel between services, significantly faster than JSON/REST. diff --git a/TRAE-Skills/code_management/Advanced_Dependency_Management_Dependabot_Renovate.md b/TRAE-Skills/code_management/Advanced_Dependency_Management_Dependabot_Renovate.md new file mode 100644 index 0000000000000000000000000000000000000000..9b0dd0b0aa8100d4298775829b6301b120fd08fa --- /dev/null +++ b/TRAE-Skills/code_management/Advanced_Dependency_Management_Dependabot_Renovate.md @@ -0,0 +1,136 @@ +# Skill: Advanced Dependency Management (Dependabot & Renovate) + +## Purpose +To automate dependency updates, reduce security risks, and keep dependencies fresh without manual toil. + +## When to Use +- When you have outdated dependencies with security vulnerabilities +- For large projects with many npm/pip/go dependencies +- When you want to reduce manual dependency debt +- For teams that want consistent dependency updates (not just security patches) + +## Procedure + +### 1. GitHub Dependabot Setup +Automate security and version updates. + +```yaml +# .github/dependabot.yml +version: 2 +updates: + # npm dependencies + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 5 + # Group related updates + groups: + eslint: + patterns: + - "eslint*" + - "@typescript-eslint/*" + # Ignore certain packages + ignore: + - dependency-name: "react" + versions: ["19.x" # Skip major version 19 for now + reviewers: + - "your-team-name" + labels: + - "dependencies" + # GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" +``` + +### 2. Renovate Setup (More Powerful) +More configurable than Dependabot. + +```json +// renovate.json +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ], + "schedule": ["before 3am on Monday"], + "packageRules": [ + { + "matchUpdateTypes": ["minor", "patch"], + "groupName": "minor and patch dependencies", + "automerge": true, + "automergeType": "branch" + }, + { + "matchDatasources": ["npm"], + "matchPackagePatterns": ["^@types/"], + "groupName": "type definitions" + } + ], + "prHourlyLimit": 2, + "prConcurrentLimit": 5, + "vulnerabilityAlerts": { + "enabled": true, + "labels": ["security"] + }, + "lockFileMaintenance": { + "enabled": true, + "schedule": ["before 4am on the first day of the month"] + } +} +``` + +### 3. Manual Dependency Auditing +Check for vulnerabilities manually (npm example. + +```bash +# npm audit +npm audit +npm audit fix # Auto-fix compatible issues +npm audit fix --force # Force fix (careful!) +# yarn audit +yarn audit +# pnpm audit +pnpm audit +# Check outdated +npm outdated +yarn outdated +pnpm outdated +# Update interactively +npm update --interactive +``` + +### 4. Pin Dependencies Correctly +Use lock files and pinning strategies. + +```json +// package.json examples +{ + "dependencies": { + "react": "^18.2.0", // Caret: minor/patch updates + "lodash": "~4.17.21", // Tilde: patch only + "axios": "1.7.2", // Exact: no auto-updates (safe!) + "next": "14.2.5" + }, + "devDependencies": { + "typescript": "5.5.3" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=9.0.0" + }, + "packageManager": "pnpm@9.5.0" // Specify package manager (Node 16+) +} +``` + +## Best Practices +- **Lock Files**: Always commit `package-lock.json`, `yarn.lock`, or `pnpm-lock.yaml` +- **Exact Versions**: Prefer exact versions for production +- **Group Updates**: Group related dependencies to reduce PR noise +- **Automerge**: Automerge minor/patch if tests pass +- **Review**: Always review major version updates carefully +- **Schedule**: Run updates weekly/monthly to avoid debt +- **Security Patches**: Prioritize security patches immediately diff --git a/TRAE-Skills/code_management/Branch_Protection_Rules.md b/TRAE-Skills/code_management/Branch_Protection_Rules.md new file mode 100644 index 0000000000000000000000000000000000000000..ecbef122546a7169c6e15adbad483cb9620ba250 --- /dev/null +++ b/TRAE-Skills/code_management/Branch_Protection_Rules.md @@ -0,0 +1,11 @@ +# Branch Protection Rules + +## Overview +Protect critical branches (main, develop) from accidental deletion or bad code. + +## Key Settings (GitHub/GitLab) +1. **Require Pull Request Reviews**: Require at least 1 or 2 approvals before merging. +2. **Require Status Checks**: Ensure CI pipelines (tests, lint) pass before merging. +3. **Require Signed Commits**: Verify commit signatures. +4. **Prevent Force Pushes**: Disable force pushing to protected branches. +5. **Include Administrators**: Apply rules to admins as well. diff --git a/TRAE-Skills/code_management/CHANGELOG_Conventional_Changelog_Auto_Generation.md b/TRAE-Skills/code_management/CHANGELOG_Conventional_Changelog_Auto_Generation.md new file mode 100644 index 0000000000000000000000000000000000000000..80981de2d22f84992689b513cfd9612e35b1c445 --- /dev/null +++ b/TRAE-Skills/code_management/CHANGELOG_Conventional_Changelog_Auto_Generation.md @@ -0,0 +1,107 @@ +# Skill: CHANGELOG Conventional Changelog Auto-Generation + +## Purpose +To keep a clear, human-readable CHANGELOG that tells users what changed in each release. + +## When to Use +- For libraries/apps with users (internal or external) +- When you want to automate release notes +- For teams that forget to update CHANGELOGs + +## Procedure + +### 1. Keep a CHANGELOG +Follow the Keep a CHANGELOG format. + +```markdown +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [1.2.0] - 2024-06-14 +### Added +- Dark mode toggle +- User profile page + +### Fixed +- Payment timeout issue +- Broken links in docs + +### Changed +- Updated dependencies + +## [1.1.0] - 2024-05-20 +### Added +- Cart persistence +``` + +### 2. Auto-Generate with Conventional Changelog +Use Conventional Commits to auto-generate. + +```bash +# Install dependencies +npm install -D standard-version +# Add script +``` + +```json +{ + "scripts": { + "release": "standard-version" + } +} +``` + +```bash +# First release +npm run release -- --first-release +# Patch release (fixes) +npm run release +# Minor release (features) +npm run release -- --release-as minor +# Major release (breaking) +npm run release -- --release-as major +``` + +### 3. GitHub Release Notes +Auto-generate GitHub releases with Release Drafter. + +```yaml +# .github/release-drafter.yml +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +categories: + - title: '🚀 Features' + labels: + - 'feature' + - title: '🐛 Bug Fixes' + labels: + - 'fix' + - title: '📝 Documentation' + labels: + - 'docs' +version-resolver: + major: + labels: + - 'breaking' + minor: + labels: + - 'feature' + patch: + labels: + - 'fix' + - 'chore' +``` + +## Best Practices +- **Keep a CHANGELOG**: Follow the Keep a CHANGELOG format +- **Group Changes**: Group by Added/Changed/Fixed/Removed +- **Link Versions**: Link versions to compare URLs +- **Date Releases**: Always add release dates +- **Semantic Versioning**: Use SemVer with Conventional Commits +- **Auto-Generate**: Use tools to reduce manual work +- **Human-Readable**: Make it easy for users (not just devs!) diff --git a/TRAE-Skills/code_management/Code_Documentation_Standards_JSDoc_TSDoc.md b/TRAE-Skills/code_management/Code_Documentation_Standards_JSDoc_TSDoc.md new file mode 100644 index 0000000000000000000000000000000000000000..b32f07ebe7ec48facb0134cae8e28e66ab136b8a --- /dev/null +++ b/TRAE-Skills/code_management/Code_Documentation_Standards_JSDoc_TSDoc.md @@ -0,0 +1,148 @@ +# Skill: Code Documentation Standards (JSDoc/TSDoc) + +## Purpose +To write clear, consistent, and useful documentation for your code that helps teammates (and future you!). + +## When to Use +- When working on team projects +- For public/open-source libraries +- When you want to reduce onboarding time +- For code that will be maintained long-term + +## Procedure + +### 1. JSDoc/TSDoc Basics +Document functions, classes, and types. + +```typescript +/** + * Calculates the total price of items in a cart, applying discounts and taxes. + * @param items - Array of cart items with id, price, and quantity + * @param discount - Discount percentage (0-100) + * @param taxRate - Tax rate (e.g., 0.08 for 8%) + * @returns Total price after discount and tax + * @throws {Error} If discount is >100 or <0 + * @example + * ```typescript + * const cart = [{ id: "1", price: 10, quantity: 2 }]; + * const total = calculateTotal(cart, 10, 0.08); + * console.log(total); // 19.44 + * ``` + */ +function calculateTotal( + items: { id: string; price: number; quantity: number }[], + discount: number, + taxRate: number +): number { + if (discount < 0 || discount > 100) throw new Error("Invalid discount"); + const subtotal = items.reduce((sum, item) => sum + item.price * item.quantity, 0); + const afterDiscount = subtotal * (1 - discount / 100); + return afterDiscount * (1 + taxRate); +} + +/** + * A user in our system. + * @typedef {Object} User + * @property {string} id - Unique user ID + * @property {string} name - Full name + * @property {string} email - Email address + * @property {Date} createdAt - Account creation date + */ + +/** + * User service class for managing users. + */ +class UserService { + /** + * Creates a new user. + * @param userData - User data without ID/createdAt + * @returns Created user + */ + async createUser(userData: Omit): Promise { + // ... + } +} +``` + +### 2. React Component Documentation +Document React components. + +```tsx +/** + * A reusable button component with different variants. + * @component + * @example + * ```tsx + * + * ``` + * @param {Object} props - Component props + * @param {("primary" | "secondary" | "danger"} props.variant - Button style variant + * @param {boolean} [props.disabled=false] - Whether button is disabled + * @param {React.ReactNode} props.children - Button content + * @param {() => void} [props.onClick] - Click handler + */ +function Button({ + variant, + disabled = false, + children, + onClick, +}: { + variant: "primary" | "secondary" | "danger"; + disabled?: boolean; + children: React.ReactNode; + onClick?: () => void; +}) { + return ( + + ); +} +``` + +### 3. README for Repositories +Standardize your repo README. + +```markdown +# Project Name +Short, descriptive subtitle. + +## Features +- ✨ Feature 1 +- 🚀 Feature 2 + +## Installation +```bash +npm install +``` + +## Usage +```typescript +import { thing } from "package"; +const result = thing(); +``` + +## Contributing +See [CONTRIBUTING.md](./CONTRIBUTING.md). + +## License +MIT +``` + +## Best Practices +- **Why, Not What**: Explain *why* code exists, not just what it does +- **Update Docs**: Keep docs in sync with code changes +- **Examples**: Always add usage examples +- **TS over JSDoc**: Prefer TypeScript types + TSDoc for TS projects +- **Avoid Redundancy**: Don't document obvious things +- **Docstrings**: Use docstrings for public APIs +- **README**: Always have a good README diff --git a/TRAE-Skills/code_management/Code_Review_Guidelines.md b/TRAE-Skills/code_management/Code_Review_Guidelines.md new file mode 100644 index 0000000000000000000000000000000000000000..d230c2ff893738bd444ca65c3ad8e550b8eda79d --- /dev/null +++ b/TRAE-Skills/code_management/Code_Review_Guidelines.md @@ -0,0 +1,15 @@ +# Code Review Guidelines + +## Overview +Effective code reviews improve code quality and facilitate knowledge sharing. + +## For Reviewers +- **Be constructive**: Focus on the code, not the person. +- **Check logic**: Look for bugs, edge cases, and security issues. +- **Style**: Let automated tools handle formatting. +- **Response time**: Review within 24 hours if possible. + +## For Authors +- **Context**: Provide a clear description and screenshots/videos if UI changes. +- **Small PRs**: Keep changes focused and small (< 400 lines). +- **Self-review**: Review your own code before assigning others. diff --git a/TRAE-Skills/code_management/Dead_Code_Elimination.md b/TRAE-Skills/code_management/Dead_Code_Elimination.md new file mode 100644 index 0000000000000000000000000000000000000000..82b5933c3aa15a5d66e8fb937b64cda4a7a4b3ad --- /dev/null +++ b/TRAE-Skills/code_management/Dead_Code_Elimination.md @@ -0,0 +1,42 @@ +# Dead Code Elimination + +## Overview +Removing unused code reduces bundle size and maintenance burden. + +## Procedure + +### 1. Automated Detection (Knip) +Knip is a powerful tool to find unused files, dependencies, and exports. + +```bash +# Install +npm install --save-dev knip + +# Run analysis +npx knip +``` + +### 2. TypeScript-Specific Audit (ts-prune) +Find unused exports in your TypeScript files. + +```bash +# Run directly with npx +npx ts-prune | grep -v "(used in module)" +``` + +### 3. Bundler Tree Shaking +Ensure your `package.json` and bundler config (Vite/Webpack) support tree shaking. +- Set `"sideEffects": false` in `package.json` for libraries that don't have side effects. +- Use ES Modules (`import/export`) instead of CommonJS (`require`). + +### 4. Manual Review +- **Search for Usage**: Before deleting, use `grep` or IDE search to verify no dynamic references exist. +- **Delete and Test**: Delete the code, run the full test suite, and check for runtime errors. + +## Constraints +- **Side Effects**: Be careful with code that has side effects (e.g., global polyfills, CSS imports). +- **Dynamic Access**: Tools might miss code accessed via dynamic strings (e.g., `eval` or `object[someVar]`). +- **Dependencies**: Don't just delete code; also remove unused packages from `package.json` (`npm uninstall`). + +## Expected Output +A leaner codebase with reduced bundle sizes, faster build times, and less "noise" for developers to maintain. diff --git a/TRAE-Skills/code_management/Dependency_Pinning_Reproducible_Builds.md b/TRAE-Skills/code_management/Dependency_Pinning_Reproducible_Builds.md new file mode 100644 index 0000000000000000000000000000000000000000..012d8d0a9f23e65c9d45a6113051f69fe84f5adc --- /dev/null +++ b/TRAE-Skills/code_management/Dependency_Pinning_Reproducible_Builds.md @@ -0,0 +1,97 @@ +# Skill: Dependency Pinning & Reproducible Builds + +## Purpose +To ensure your project builds the same way for everyone, every time. + +## When to Use +- When "it works on my machine" issues +- For CI/CD pipelines +- For open-source projects +- When you need consistent builds + +## Procedure + +### 1. Lock Files +Always commit lock files. + +```bash +# npm: package-lock.json +# yarn: yarn.lock +# pnpm: pnpm-lock.yaml +# Python: Pipfile.lock, requirements.txt (with hashes) +# Go: go.sum +``` + +### 2. Exact Versions +Pin versions in package.json (npm example. + +```json +{ + "dependencies": { + "react": "18.2.0", // Exact version (no ^/~) + "lodash": "4.17.21", + "next": "14.2.5" + }, + "devDependencies": { + "typescript": "5.5.3" + }, + "engines": { + "node": "20.15.0", // Exact Node version + "pnpm": "9.5.0" + }, + "packageManager": "pnpm@9.5.0" // Enforce package manager +} +``` + +### 3. .nvmrc / .tool-versions +Specify Node version. + +```bash +# .nvmrc +20.15.0 +``` + +``` +# .tool-versions (for asdf) +nodejs 20.15.0 +python 3.12.0 +``` + +### 4. npm ci (Clean Install) +Use `npm ci` instead of `npm install` in CI. + +```bash +# npm: npm ci installs *exact* versions from lock file +npm ci +# pnpm: pnpm install --frozen-lockfile +pnpm install --frozen-lockfile +# yarn: yarn install --frozen-lockfile +yarn install --frozen-lockfile +``` + +### 5. Docker for Reproducibility +Use Docker to freeze the entire environment. + +```dockerfile +# Use exact base image +FROM node:20.15.0-alpine +WORKDIR /app +# Copy lock file first for caching +COPY package.json pnpm-lock.yaml ./ +# Install dependencies +RUN pnpm install --frozen-lockfile +# Copy code +COPY . . +# Build +RUN pnpm build +# Run +CMD ["pnpm", "start"] +``` + +## Best Practices +- **Commit Lock Files**: Always commit lock files to repo +- **Exact Versions**: Use exact versions for production +- **Enforce Package Manager**: Use `packageManager` field +- **Use `npm ci`**: Use in CI instead of `npm install` +- **Dockerize**: Use Docker for consistent environments +- **Cache Dependencies**: Cache dependencies in CI to speed up builds diff --git a/TRAE-Skills/code_management/ESLint_Prettier_Setup.md b/TRAE-Skills/code_management/ESLint_Prettier_Setup.md new file mode 100644 index 0000000000000000000000000000000000000000..1c98a8670d245beade20370345643c8a3bbbbcbc --- /dev/null +++ b/TRAE-Skills/code_management/ESLint_Prettier_Setup.md @@ -0,0 +1,19 @@ +# ESLint & Prettier Setup + +## Overview +Consistent code style and catching errors early are essential for maintainability. + +## Implementation +1. **Install**: + ```bash + npm install --save-dev eslint prettier + ``` +2. **Config**: + - Create `.eslintrc.json` or `eslint.config.js`. + - Create `.prettierrc`. +3. **Integration**: + - Use `eslint-config-prettier` to disable ESLint rules that conflict with Prettier. + +## Usage +- Run `eslint . --fix` to auto-fix linting issues. +- Run `prettier --write .` to format code. diff --git a/TRAE-Skills/code_management/Git_Commit_Convention_Conventional_Commits.md b/TRAE-Skills/code_management/Git_Commit_Convention_Conventional_Commits.md new file mode 100644 index 0000000000000000000000000000000000000000..a1bfd73e605be253faa322af036b48045a1c37ad --- /dev/null +++ b/TRAE-Skills/code_management/Git_Commit_Convention_Conventional_Commits.md @@ -0,0 +1,20 @@ +# Git Commit Convention (Conventional Commits) + +## Overview +Standardizing commit messages is crucial for automated changelog generation and clear project history. + +## Best Practices +- **Format**: `type(scope): subject` +- **Types**: + - `feat`: New feature + - `fix`: Bug fix + - `docs`: Documentation only + - `style`: Formatting, missing semi colons, etc; no code change + - `refactor`: Refactoring production code + - `test`: Adding tests, refactoring test; no production code change + - `chore`: Updating build tasks, package manager configs, etc + +## Example +```bash +git commit -m "feat(auth): implement jwt token handling" +``` diff --git a/TRAE-Skills/code_management/Git_Hooks_Husky.md b/TRAE-Skills/code_management/Git_Hooks_Husky.md new file mode 100644 index 0000000000000000000000000000000000000000..ed0686be3a731a366613ded87e045da2c69d0607 --- /dev/null +++ b/TRAE-Skills/code_management/Git_Hooks_Husky.md @@ -0,0 +1,20 @@ +# Git Hooks with Husky + +## Overview +Automate tasks like linting and testing before commits or pushes to ensure code quality. + +## Setup +1. **Install Husky**: + ```bash + npm install --save-dev husky + npx husky install + ``` +2. **Add Hook**: + ```bash + npx husky add .husky/pre-commit "npm run lint" + ``` + +## Common Hooks +- `pre-commit`: Run linter, type check. +- `commit-msg`: Verify commit message format (e.g., using commitlint). +- `pre-push`: Run unit tests. diff --git a/TRAE-Skills/code_management/Git_Rebase_Merge_Squash_Guidelines.md b/TRAE-Skills/code_management/Git_Rebase_Merge_Squash_Guidelines.md new file mode 100644 index 0000000000000000000000000000000000000000..2aa21b600ce833fad8a3db5eae155e81c8ff75da --- /dev/null +++ b/TRAE-Skills/code_management/Git_Rebase_Merge_Squash_Guidelines.md @@ -0,0 +1,72 @@ +# Skill: Git Rebase, Merge & Squash Guidelines + +## Purpose +To choose the right Git operation for clean history and clear merges. + +## When to Use +- When cleaning up commit history +- For merging feature branches +- When you want a clean main branch +- For teams confused about rebase/merge + +## Procedure + +### 1. When to Use `git merge --no-ff +Preserve branch history (good for GitFlow). + +```bash +# Merge feature branch to main, preserving history +git checkout main +git pull +git merge --no-ff feature/my-feature +# Result: Merge commit + all feature commits +``` + +### 2. When to Use `git rebase` +Clean up history before merging (good for Trunk-Based). + +```bash +# Rebase feature branch on top of main +git checkout feature/my-feature +git fetch origin +git rebase origin/main +# Fix conflicts if any +git add . +git rebase --continue +# Push (force-with-lease for safety +git push --force-with-lease +``` + +### 3. When to Use Squash Merge +Squash all feature commits into one (clean main history). + +```bash +# GitHub/GitLab: "Squash and merge" option in PR +# Result: One clean commit on main +# Example commit message: feat: add dark mode (closes #123) +``` + +### 4. Interactive Rebase +Clean up your commit history. + +```bash +git rebase -i HEAD~5 # Edit last 5 commits +# In editor: +# pick = keep commit +# squash = combine with previous +# reword = edit commit message +# fixup = combine without message +# Example: +pick abc123 First commit +squash def456 Fix typo +squash ghi789 Add tests +# Result: One commit combining all changes +``` + +## Best Practices +- **Never Rebase Public**: Never rebase shared branches (main/develop) +- **Force-With-Lease**: Use `--force-with-lease` instead of `--force` +- **Squash Feature Commits**: Squash small, related commits for clean history +- **Merge Main Branches**: Use `--no-ff` for GitFlow releases +- **Rebase Feature**: Rebase feature branches on main often to avoid conflicts +- **Commit Messages**: Write good commit messages after squashing diff --git a/TRAE-Skills/code_management/Git_Workflow_Strategies.md b/TRAE-Skills/code_management/Git_Workflow_Strategies.md new file mode 100644 index 0000000000000000000000000000000000000000..f7c9232005d4501704ce4bbdd6afd02246b46924 --- /dev/null +++ b/TRAE-Skills/code_management/Git_Workflow_Strategies.md @@ -0,0 +1,94 @@ +# Skill: Git Workflow Strategies (GitFlow, Trunk-Based, GitHub Flow) + +## Purpose +To choose and implement the right Git workflow for your team, ensuring smooth collaboration, releases. + +## When to Use +- When onboarding a new team to Git +- For teams working on different release cadences +- When switching from ad-hoc commits to structured workflows +- For teams using CI/CD pipelines +- When you need clarity on branching, merging, and releasing + +## Procedure + +### 1. GitHub Flow (Simplest Workflow +Perfect for continuous deployment. + +```bash +# 1. Create a feature branch from main +git checkout -b feature/new-checkout +# 2. Commit changes +git add . +git commit -m "feat: add new checkout flow" +# 3. Push branch +git push -u origin feature/new-checkout +# 4. Open Pull Request (PR) +# 5. Review, discuss, update PR +# 6. Merge PR to main +# 7. Deploy immediately +git checkout main +git pull +git branch -d feature/new-checkout # Cleanup +``` + +### 2. GitFlow (For Scheduled Releases) +Ideal for teams with regular release cycles (e.g., monthly). + +```bash +# 1. Main branches: main (production) and develop +git checkout -b develop # Start from develop for new features +# 2. Feature branch from develop +git checkout -b feature/user-profile develop +git commit -m "feat: add user profile page" +git push origin feature/user-profile +# 3. Merge feature to develop via PR +# 4. When ready for release: create release branch from develop +git checkout -b release/v1.2.0 develop +# 5. Fix bugs on release branch +git commit -m "fix: adjust pricing calculation" +# 6. Merge release to main AND develop +git checkout main +git merge --no-ff release/v1.2.0 +git tag -a v1.2.0 -m "Release v1.2.0" +git checkout develop +git merge --no-ff release/v1.2.0 +# 7. Delete release branch +git branch -d release/v1.2.0 +# 8. Hotfix from main +git checkout -b hotfix/critical-bug main +git commit -m "fix: resolve payment timeout" +git checkout main +git merge --no-ff hotfix/critical-bug +git tag -a v1.2.1 -m "Hotfix v1.2.1" +git checkout develop +git merge --no-ff hotfix/critical-bug +git branch -d hotfix/critical-bug +``` + +### 3. Trunk-Based Development (High-Velocity Teams) +For teams that deploy multiple times per day. + +```bash +# 1. Work directly on short-lived feature branches (max 1-2 days) +git checkout -b feat/fast-feature main +# 2. Small, focused commits +git commit -m "feat: add button hover" +# 3. Push and merge to main quickly (via PR with tests!) +# 4. Use feature flags for incomplete features +# Example: Feature flag in code +if (featureFlags.newDashboard) { + renderNewDashboard(); +} +# 5. Clean up feature flags once released +``` + +## Best Practices +- **GitHub Flow**: Use for startups/teams with continuous deployment +- **GitFlow**: Use for scheduled releases (e.g., enterprise software) +- **Trunk-Based**: Use for high-velocity teams with strong CI/CD +- **Short-Lived Branches**: Keep branches < 1-2 days max +- **Small Commits**: Atomic, focused commits for easier review/rollback +- **CI on PR**: Always run tests before merging +- **Cleanup**: Delete branches after merging +- **Tags**: Use semantic tags for releases diff --git a/TRAE-Skills/code_management/Managing_Technical_Debt.md b/TRAE-Skills/code_management/Managing_Technical_Debt.md new file mode 100644 index 0000000000000000000000000000000000000000..532a280a5617265d84ebec08155fe893ae64fc46 --- /dev/null +++ b/TRAE-Skills/code_management/Managing_Technical_Debt.md @@ -0,0 +1,10 @@ +# Managing Technical Debt + +## Overview +Technical debt is inevitable but must be managed to prevent project stagnation. + +## Strategies +1. **Identification**: Track debt items in the backlog (e.g., with a "tech-debt" label). +2. **Allocation**: Dedicate a percentage of time (e.g., 20%) in each sprint to addressing debt. +3. **Boy Scout Rule**: "Leave the code better than you found it." Refactor slightly when touching a file. +4. **Documentation**: Document *why* a shortcut was taken (e.g., `// TODO: Refactor this when X is ready`). diff --git a/TRAE-Skills/code_management/Monorepo_Setup_Turborepo.md b/TRAE-Skills/code_management/Monorepo_Setup_Turborepo.md new file mode 100644 index 0000000000000000000000000000000000000000..ab717ef8748374190d8e3c20f4de9ed11f8e057c --- /dev/null +++ b/TRAE-Skills/code_management/Monorepo_Setup_Turborepo.md @@ -0,0 +1,69 @@ +# Monorepo Setup (Turborepo) + +## Overview +Managing multiple packages or apps in a single repository can simplify dependency management and code sharing. + +## Key Concepts +- **Workspaces**: NPM/Yarn/PNPM workspaces to link packages locally. +- **Turborepo**: High-performance build system for monorepos. + +## Procedure + +### 1. Workspace Configuration +Define your project structure in `package.json` (for NPM/Yarn) or `pnpm-workspace.yaml`. + +```json +// package.json (root) +{ + "workspaces": [ + "apps/*", + "packages/*" + ] +} +``` + +### 2. Initialize Turborepo +Add Turbo to your project. + +```bash +npx init turbo@latest +``` + +### 3. Configuring `turbo.json` +Define the dependency graph and cacheable tasks. + +```json +{ + "$schema": "https://turbo.build/schema.json", + "pipeline": { + "build": { + "dependsOn": ["^build"], // Build dependencies first + "outputs": ["dist/**", ".next/**"] + }, + "test": { + "dependsOn": ["build"], + "outputs": [] + }, + "lint": {} + } +} +``` + +### 4. Running Commands +Turbo automatically parallelizes tasks and caches results. + +```bash +# Build all apps and packages +npx turbo run build + +# Run tests only for changed packages +npx turbo run test --filter=...[origin/main] +``` + +## Constraints +- **Circular Dependencies**: Avoid circular dependencies between packages in the monorepo. +- **Hoisting**: Be aware of how dependencies are "hoisted" to the root `node_modules`. +- **Versioning**: Decide between a "Fixed" versioning strategy (all packages same version) or "Independent" versioning. + +## Expected Output +A high-performance monorepo where builds and tests are optimized through intelligent caching and parallel execution. diff --git a/TRAE-Skills/code_management/Npm_Scripts_Automation.md b/TRAE-Skills/code_management/Npm_Scripts_Automation.md new file mode 100644 index 0000000000000000000000000000000000000000..ae2249b1dbeec29ca4f40d7381c5d765e08f5be8 --- /dev/null +++ b/TRAE-Skills/code_management/Npm_Scripts_Automation.md @@ -0,0 +1,20 @@ +# NPM Scripts Automation + +## Overview +Use `package.json` scripts to standardize common development tasks. + +## Best Practices +- **Naming**: Use standard names like `start`, `build`, `test`. +- **Composition**: Use `pre` and `post` hooks (e.g., `prebuild` runs before `build`). +- **Cross-platform**: Use tools like `rimraf` for deletion and `cross-env` for environment variables to ensure scripts work on Windows and Unix. + +## Example +```json +"scripts": { + "start": "node dist/index.js", + "dev": "nodemon src/index.ts", + "build": "tsc", + "lint": "eslint .", + "test": "jest" +} +``` diff --git a/TRAE-Skills/code_management/Pre_Commit_Pre_Push_Automation.md b/TRAE-Skills/code_management/Pre_Commit_Pre_Push_Automation.md new file mode 100644 index 0000000000000000000000000000000000000000..5de06ef620aca09f949a1cc266710160eb24e8fe --- /dev/null +++ b/TRAE-Skills/code_management/Pre_Commit_Pre_Push_Automation.md @@ -0,0 +1,101 @@ +# Skill: Pre-Commit & Pre-Push Hooks Automation + +## Purpose +To catch issues *before* commits/pushes to CI, saving time. + +## When to Use +- When you forget to lint/format before committing +- For teams that want consistent code quality +- When you want to reduce CI failures + +## Procedure + +### 1. Pre-Commit with Husky + lint-staged +Run linters/formatters on staged files. + +```bash +# Install dependencies +npm install -D husky lint-staged +# Initialize Husky +npx husky install +# Add pre-commit hook +npx husky add .husky/pre-commit "npx lint-staged" +# Make executable (Windows) +npx husky install +``` + +```json +// package.json +{ + "lint-staged": { + "*.{js,jsx,ts,tsx}": [ + "eslint --fix", + "prettier --write" + ], + "*.{json,css,md}": [ + "prettier --write" + ] + } +} +``` + +### 2. Pre-Push Hook +Run tests before pushing. + +```bash +# Add pre-push hook +npx husky add .husky/pre-push "npm test" +``` + +### 3. Pre-Commit Framework (Python, etc.) +For non-JS projects. + +```yaml +# .pre-commit-config.yaml +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml +- repo: https://github.com/psf/black + rev: 24.4.2 + hooks: + - id: black +``` + +```bash +# Install pre-commit +pip install pre-commit +pre-commit install +``` + +### 4. Commit Message Hook +Enforce Conventional Commits. + +```bash +# .husky/commit-msg +npx --no -- commitlint --edit "$1" +``` + +```javascript +// commitlint.config.js +export default { + extends: ['@commitlint/config-conventional'], +}; +``` + +## Best Practices +- **Lint Staged**: Only lint staged files to save time +- **Fast Hooks**: Keep hooks < 10 seconds +- **Test in CI**: Also run checks in CI as a safety net +- **Allow Bypass**: Allow `--no-verify` for emergencies (document!) +- **Install Script**: Add a `prepare` script to auto-install husky +```json +{ + "scripts": { + "prepare": "husky install" + } +} +``` diff --git a/TRAE-Skills/code_management/Repository_Archiving_Legacy_Code_Retirement.md b/TRAE-Skills/code_management/Repository_Archiving_Legacy_Code_Retirement.md new file mode 100644 index 0000000000000000000000000000000000000000..19edc066f880453e89df0cd3f32843c565d8ae07 --- /dev/null +++ b/TRAE-Skills/code_management/Repository_Archiving_Legacy_Code_Retirement.md @@ -0,0 +1,74 @@ +# Skill: Repository Archiving & Legacy Code Retirement + +## Purpose +To clean up old repos, reduce clutter, and retire legacy code safely. + +## When to Use +- When you have unused repos +- For legacy code no longer in use +- When you want to reduce security risks from old code +- For offboarding projects + +## Procedure + +### 1. Archive a Repository (GitHub/GitLab) +Archive a repo that's no longer active. + +```bash +# 1. Update README: Add notice at top +# [ARCHIVED] This project is no longer maintained. +# 2. Close all open issues/PRs +# 3. Archive the repo (GitHub: Settings → Danger Zone → Archive this repository) +# 4. Keep it read-only (no pushes/merges) +``` + +### 2. Retire Legacy Code in Active Repo +Remove legacy code in a live repo. + +```bash +# 1. Mark as deprecated +/** + * @deprecated Use newUserService instead + */ +function oldUserService() { + console.warn('oldUserService is deprecated!'); + // ... +} +# 2. Keep for a grace period (2-3 releases) +# 3. Remove in major version +git rm src/legacy/ +git commit -m "chore!: remove legacy user service" +``` + +### 3. Archive Old Branches +Clean up old branches. + +```bash +# List merged branches +git branch --merged main +# Delete local merged branches +git branch -d old-feature +# Delete remote merged branches +git push origin --delete old-feature +# Auto-delete branches after merging (GitHub repo settings) +``` + +### 4. Archive Old CI/CD Pipelines +Disable old pipelines. + +```yaml +# Add a notice at top of workflow +# [ARCHIVED] This workflow is no longer used +name: Old Deploy +on: + push: + branches: [old-branch] # Disable triggers +``` + +## Best Practices +- **Document**: Add archive/retirement clearly +- **Grace Period**: Give users time to migrate +- **Backup**: Make sure you have a backup before archiving +- **Notify**: Notify stakeholders before archiving +- **Clean Up**: Delete old branches/tags +- **Security**: Archive old repos to reduce attack surface diff --git a/TRAE-Skills/code_management/Repository_Structure_Organization.md b/TRAE-Skills/code_management/Repository_Structure_Organization.md new file mode 100644 index 0000000000000000000000000000000000000000..c39ab792518e044460f8c7b633bfd48c0d724b76 --- /dev/null +++ b/TRAE-Skills/code_management/Repository_Structure_Organization.md @@ -0,0 +1,98 @@ +# Skill: Repository Structure & Project Organization + +## Purpose +To structure repos consistently so anyone can find what they need quickly. + +## When to Use +- When starting a new project +- For reorganizing messy repos +- When onboarding new developers +- For multi-team repos + +## Procedure + +### 1. Standard Monolith/Polyrepo Decision +Choose between monorepo and polyrepo. + +``` +# Polyrepo (separate repos for separate services/apps +├── user-service/ # Backend service +│ ├── src/ +│ ├── tests/ +│ └── package.json +├── checkout-service/ +└── web-app/ # Frontend app + +# Monorepo (all in one repo) +├── apps/ +│ ├── web/ +│ └── admin/ +├── packages/ +│ ├── ui-components/ +│ └── utils/ +└── package.json +``` + +### 2. Frontend Project Structure +Standard Next.js/React structure. + +``` +src/ +├── app/ # Next.js App Router pages +├── components/ +│ ├── ui/ # Reusable UI components +│ └── features/ # Feature-specific components +├── lib/ # Utilities, helpers +│ ├── utils/ +│ └── api/ # API clients +├── hooks/ # Custom React hooks +├── types/ # TypeScript types +├── styles/ # Global styles +└── tests/ # Tests +``` + +### 3. Backend Project Structure +Express/NestJS structure. + +``` +src/ +├── modules/ # Feature modules +│ ├── users/ +│ │ ├── users.controller.ts +│ │ ├── users.service.ts +│ │ ├── users.module.ts +│ │ └── dto/ +├── common/ # Shared code +│ ├── guards/ +│ ├── interceptors/ +│ └── filters/ +├── config/ # Config files +├── prisma/ # Prisma schema +└── main.ts # Entry point +``` + +### 4. Root-Level Files +Always include these. + +``` +your-repo/ +├── .gitignore +├── README.md +├── CONTRIBUTING.md # How to contribute +├── LICENSE +├── package.json +├── .eslintrc.js # Lint config +├── .prettierrc # Format config +├── .github/ # GitHub-specific files +│ └── workflows/ # GitHub Actions +├── .husky/ # Git hooks +└── .env.example # Example env vars +``` + +## Best Practices +- **Consistency**: Keep structure consistent across repos +- **Naming**: Use clear, descriptive folder names +- **Colocation**: Keep related code together +- **Separation**: Separate concerns (src vs tests, etc.) +- **Flat**: Don't nest folders too deep +- **Document**: Add a small README in complex folders diff --git a/TRAE-Skills/code_management/Semantic_Versioning_Strategy.md b/TRAE-Skills/code_management/Semantic_Versioning_Strategy.md new file mode 100644 index 0000000000000000000000000000000000000000..95a7645b1e04210a92bd9ed0512d883e802ccdeb --- /dev/null +++ b/TRAE-Skills/code_management/Semantic_Versioning_Strategy.md @@ -0,0 +1,71 @@ +# Skill: Semantic Versioning Strategy + +## Purpose +To use Semantic Versioning (SemVer) as a clear and predictable way to communicate the nature and impact of code changes to consumers, stakeholders, and other developers. + +## When to Use +- When maintaining libraries, packages, APIs, or microservices +- When setting up CI/CD release pipelines +- When defining backward compatibility guarantees + +## Procedure + +### 1. Understanding the Format +Version numbers must follow the `MAJOR.MINOR.PATCH` format. + +- **MAJOR version**: When you make incompatible API changes or breaking changes. + - E.g., Changing the signature of a widely used public function. + - Resets `MINOR` and `PATCH` to 0. (e.g., `1.4.2` -> `2.0.0`) +- **MINOR version**: When you add functionality in a backward-compatible manner. + - E.g., Adding a new optional parameter, or a completely new feature that doesn't affect existing workflows. + - Resets `PATCH` to 0. (e.g., `1.4.2` -> `1.5.0`) +- **PATCH version**: When you make backward-compatible bug fixes. + - E.g., Fixing a typo, patching a security vulnerability without altering the API. + - (e.g., `1.4.2` -> `1.4.3`) + +### 2. Pre-release Tags and Build Metadata +Use pre-release tags for betas and release candidates: +- `1.0.0-alpha.1` +- `1.0.0-beta.2` +- `1.0.0-rc.1` + +Build metadata can be appended with a plus sign (ignored by SemVer precedence): +- `1.0.0+20130313144700` + +### 3. Version Zero (0.y.z) +Use major version zero for initial development. Anything MAY change at any time. The public API should not be considered stable. +- Start at `0.1.0`. +- Release `1.0.0` when the API is stable enough for production use. + +### 4. Automation Tools +Instead of manually guessing versions, automate the process based on Conventional Commits. + +**Using `semantic-release` (Node.js)** +```bash +npm install -D semantic-release +``` + +Create a `.releaserc` file: +```json +{ + "branches": ["main"], + "plugins": [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] +} +``` + +This will automatically: +1. Analyze commits (`fix:` -> PATCH, `feat:` -> MINOR, `BREAKING CHANGE:` -> MAJOR). +2. Generate a Changelog. +3. Bump the version in `package.json`. +4. Create a Git tag and GitHub Release. + +## Best Practices +- Never reuse a version number once published. If a release is botched, publish a new patch version. +- Treat documentation updates and refactors that do not affect the public API as chores or patches depending on your pipeline. \ No newline at end of file diff --git a/TRAE-Skills/devops/AWS_Lambda_Function_Design.md b/TRAE-Skills/devops/AWS_Lambda_Function_Design.md new file mode 100644 index 0000000000000000000000000000000000000000..d47ec4abe9e4942209fe4a8fa30f669e5d5290ce --- /dev/null +++ b/TRAE-Skills/devops/AWS_Lambda_Function_Design.md @@ -0,0 +1,25 @@ +# Skill: AWS Lambda Function Design + +## Purpose +To build serverless functions. + +## When to Use +- Triggers, layers, and concurrency. +- When the specific requirement for AWS Lambda Function Design arises in the project. + +## Procedure +1. **Analysis**: Understand the requirements for AWS Lambda Function Design. +2. **Implementation**: + - Step 1: Initialize necessary configurations. + - Step 2: Implement the core logic for AWS Lambda Function Design. + - Step 3: Handle edge cases and errors. +3. **Verification**: + - Test the implementation. + - Ensure it meets the project standards. + +## Constraints +- Follow existing project coding standards. +- Ensure performance and security best practices are met. + +## Expected Output +A working implementation of AWS Lambda Function Design with documentation and tests. diff --git a/TRAE-Skills/devops/AWS_Secrets_Manager_Integration.md b/TRAE-Skills/devops/AWS_Secrets_Manager_Integration.md new file mode 100644 index 0000000000000000000000000000000000000000..72ad88169c71bcc30e3de17b86cdf000621f8f49 --- /dev/null +++ b/TRAE-Skills/devops/AWS_Secrets_Manager_Integration.md @@ -0,0 +1,26 @@ +# Skill: AWS Secrets Manager Integration + +## Purpose +To retrieve secrets (database credentials, API keys) at runtime rather than storing them in environment variables or config files, enabling automatic rotation and better security. + +## When to Use +- High-security environments. +- When key rotation is required by compliance. + +## Procedure +1. **Store Secret**: Create a secret in AWS Secrets Manager (JSON key/value). +2. **IAM Role**: Grant the application's IAM Role permission to `secretsmanager:GetSecretValue`. +3. **Application Code**: + - Use AWS SDK (`@aws-sdk/client-secrets-manager`). + - On startup, fetch the secret by ID. + - Parse JSON and set config values. +4. **Caching**: Cache the secret in memory for a short duration to avoid hitting API limits and latency costs. +5. **Rotation**: (Advanced) Configure Lambda to rotate the secret (e.g., change DB password) automatically. + +## Constraints +- Handle network failures when fetching secrets. +- Do not log the retrieved secrets. +- Fallback strategy (if service is down) - usually fail hard for security. + +## Expected Output +An application that boots up by fetching its configuration securely from a managed vault service. diff --git a/TRAE-Skills/devops/Ansible_Playbook_Creation.md b/TRAE-Skills/devops/Ansible_Playbook_Creation.md new file mode 100644 index 0000000000000000000000000000000000000000..387097c0d368d38e5fc65ed06e816cc85aadea42 --- /dev/null +++ b/TRAE-Skills/devops/Ansible_Playbook_Creation.md @@ -0,0 +1,87 @@ +# Skill: Ansible Playbook Creation + +## Purpose +To automate server configuration. + +## When to Use +- Tasks, handlers, and roles. +- When the specific requirement for Ansible Playbook Creation arises in the project. + +## Procedure + +### 1. Inventory Configuration +Define the target hosts in an `inventory.ini` or `inventory.yml` file. + +```ini +[webservers] +web1.example.com ansible_host=192.168.1.10 +web2.example.com ansible_host=192.168.1.11 + +[dbservers] +db1.example.com ansible_host=192.168.1.20 +``` + +### 2. Creating a Playbook +Create a `site.yml` file to define the tasks for each group. + +```yaml +--- +- name: Configure Web Servers + hosts: webservers + become: yes # Run as root + vars: + http_port: 80 + max_clients: 200 + + tasks: + - name: Ensure Nginx is installed + apt: + name: nginx + state: present + update_cache: yes + + - name: Copy Nginx configuration + template: + src: templates/nginx.conf.j2 + dest: /etc/nginx/nginx.conf + notify: + - Restart Nginx + + handlers: + - name: Restart Nginx + service: + name: nginx + state: restarted +``` + +### 3. Using Roles for Scalability +Organize playbooks into roles for reusability. + +```text +roles/ + common/ + tasks/main.yml + webserver/ + tasks/main.yml + templates/nginx.conf.j2 + handlers/main.yml +``` + +### 4. Running the Playbook +Use `ansible-playbook` to execute the automation. + +```bash +# Check syntax +ansible-playbook site.yml --syntax-check + +# Run the playbook +ansible-playbook -i inventory.ini site.yml +``` + +## Constraints +- **Idempotency**: Ensure tasks can run multiple times without changing the result if the system is already in the desired state. +- **Security**: Use `ansible-vault` to encrypt sensitive variables (passwords, API keys). +- **Modularity**: Break large playbooks into smaller roles. + +## Expected Output +A set of YAML files that automatically configure remote servers to a desired state in a reproducible way. diff --git a/TRAE-Skills/devops/Automated_Database_Backups.md b/TRAE-Skills/devops/Automated_Database_Backups.md new file mode 100644 index 0000000000000000000000000000000000000000..44ed99b454dfcd994eea14a62e1fde85bccc5deb --- /dev/null +++ b/TRAE-Skills/devops/Automated_Database_Backups.md @@ -0,0 +1,24 @@ +# Skill: Automated Database Backups + +## Purpose +To ensure data is preserved and recoverable in case of corruption, accidental deletion, or catastrophic failure. + +## When to Use +- Always. For every production database. + +## Procedure +1. **Scripting**: Write a script (bash/python) to run `pg_dump` (Postgres) or `mysqldump`. + - `pg_dump -h $HOST -U $USER $DB | gzip > backup.sql.gz`. +2. **Upload**: Stream or upload the file to offsite storage (AWS S3, Google Cloud Storage). + - `aws s3 cp backup.sql.gz s3://my-backups/$(date +%F)/`. +3. **Scheduling**: Add to crontab: `0 2 * * * /path/to/backup.sh` (Runs at 2 AM). +4. **Retention Policy**: Configure S3 Lifecycle Rules to delete backups older than X days. +5. **Verification**: **Crucial**. Regularly attempt to restore a backup to a test DB to ensure it works. + +## Constraints +- Encrypt backups at rest. +- Do not store backups on the same server as the database. +- Monitor the backup job (alert if it fails). + +## Expected Output +A reliable, automated system that produces daily/hourly backups stored securely offsite. diff --git a/TRAE-Skills/devops/Azure_Functions_Basics.md b/TRAE-Skills/devops/Azure_Functions_Basics.md new file mode 100644 index 0000000000000000000000000000000000000000..4e0306f2b3a6595081b3e7cd5364236e9d6c992b --- /dev/null +++ b/TRAE-Skills/devops/Azure_Functions_Basics.md @@ -0,0 +1,71 @@ +# Skill: Azure Functions Basics + +## Purpose +To use serverless on Azure. + +## When to Use +- Bindings and triggers. +- When the specific requirement for Azure Functions Basics arises in the project. + +## Procedure + +### 1. Project Initialization +Create a new Azure Functions project using the CLI. + +```bash +# Install tools +npm install -g azure-functions-core-tools@4 + +# Initialize project (Node.js/TypeScript) +func init MyFunctionProj --typescript +cd MyFunctionProj + +# Create a specific function (HTTP Trigger) +func new --name MyHttpTrigger --template "HTTP trigger" --authlevel "anonymous" +``` + +### 2. Defining Triggers and Bindings +Edit `function.json` (or use decorators in v4+) to define how the function is invoked. + +```typescript +// v4 Node.js model (index.ts) +import { app, HttpRequest, HttpResponseInit, InvocationContext } from "@azure/functions"; + +export async function MyHttpTrigger(request: HttpRequest, context: InvocationContext): HttpResponseInit { + context.log(`Http function processed request for url "${request.url}"`); + + const name = request.query.get('name') || await request.text() || 'world'; + + return { body: `Hello, ${name}!` }; +}; + +app.http('MyHttpTrigger', { + methods: ['GET', 'POST'], + authLevel: 'anonymous', + handler: MyHttpTrigger +}); +``` + +### 3. Local Testing +Run the function locally before deploying. + +```bash +npm start +# Function will be available at http://localhost:7071/api/MyHttpTrigger +``` + +### 4. Deployment +Deploy to Azure using the CLI or GitHub Actions. + +```bash +# Via CLI +func azure functionapp publish +``` + +## Constraints +- **Cold Starts**: Be aware of latency for the first request after inactivity on "Consumption" plans. +- **Statelessness**: Functions must be stateless. Use Azure Durable Functions if you need state/orchestration. +- **Execution Limit**: Consumption plan functions have a default timeout of 5 minutes (max 10). + +## Expected Output +A serverless function deployed to Azure that responds to triggers (HTTP, Timer, Queue) without managing underlying server infrastructure. diff --git a/TRAE-Skills/devops/Blue_Green_Deployment_Strategy.md b/TRAE-Skills/devops/Blue_Green_Deployment_Strategy.md new file mode 100644 index 0000000000000000000000000000000000000000..488f781625487249127616516a5b89cd623c61d2 --- /dev/null +++ b/TRAE-Skills/devops/Blue_Green_Deployment_Strategy.md @@ -0,0 +1,25 @@ +# Skill: Blue-Green Deployment Strategy + +## Purpose +To release new software versions with zero downtime and instant rollback capability by maintaining two identical environments (Blue and Green). + +## When to Use +- Mission-critical applications where downtime is unacceptable. +- Complex updates that might need immediate reversion. + +## Procedure +1. **Current State**: "Blue" is live (v1), serving all traffic. +2. **Deploy**: Deploy "Green" (v2) alongside Blue. Green is idle or internal-only. +3. **Test**: Run smoke tests against the Green environment. +4. **Switch Traffic**: Update the Load Balancer / Router to point to Green. +5. **Monitor**: Watch for errors. + - **Success**: Terminate Blue (or keep as backup). + - **Failure**: Switch Load Balancer back to Blue immediately. + +## Constraints +- Database schema changes must be backward compatible (Blue and Green share the DB during transition). +- Sessions might be lost if not stored in a shared store (Redis). +- Requires double resource capacity during deployment. + +## Expected Output +A deployment process definition that allows switching versions without user interruption. diff --git a/TRAE-Skills/devops/CI_Pipeline_GitHub_Actions.md b/TRAE-Skills/devops/CI_Pipeline_GitHub_Actions.md new file mode 100644 index 0000000000000000000000000000000000000000..371998040a631d16a8bcde8d2aa6068941e9e80c --- /dev/null +++ b/TRAE-Skills/devops/CI_Pipeline_GitHub_Actions.md @@ -0,0 +1,106 @@ +# Skill: CI Pipeline Setup (GitHub Actions) + +## Purpose +To automate the testing, building, and linting process whenever code is pushed, ensuring quality control and preventing regressions before merging. + +## When to Use +- When setting up a new repository. +- When enforcing code quality standards (Lint, Prettier). +- When automating deployment (CD) or release management. + +## Procedure + +### 1. Workflow Configuration (Node.js) +Create `.github/workflows/ci.yml` in your repository. + +```yaml +name: CI Pipeline + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +jobs: + build-and-test: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [18.x, 20.x] + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' # Speeds up builds by caching node_modules + + - name: Install Dependencies + run: npm ci + + - name: Run Lint + run: npm run lint + + - name: Run Tests + run: npm test -- --coverage + + - name: Upload Coverage + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} +``` + +### 2. Handling Environment Secrets +Never hardcode secrets. Use GitHub Repository Secrets. + +```yaml + - name: Run Integration Tests + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + TEST_API_KEY: ${{ secrets.TEST_API_KEY }} + run: npm run test:integration +``` + +### 3. Service Containers (Databases) +Run a database alongside your tests using Docker services. + +```yaml + services: + redis: + image: redis + ports: + - 6379:6379 + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: password + ports: + - 5432:5432 +``` + +### 4. Build Artifacts +Save build outputs for later deployment stages. + +```yaml + - name: Build Application + run: npm run build + + - name: Upload Build Artifact + uses: actions/upload-artifact@v4 + with: + name: build-output + path: dist/ +``` + +## Constraints +- **Security**: Do not log sensitive environment variables. Use masked secrets. +- **Efficiency**: Use caching for `npm` or `yarn` to keep build times under 5 minutes. +- **Reliability**: Ensure tests run in a clean environment every time. + +## Expected Output +A fully functional GitHub Action that automatically validates every pull request and push to protected branches. diff --git a/TRAE-Skills/devops/Chaos_Engineering_Basics.md b/TRAE-Skills/devops/Chaos_Engineering_Basics.md new file mode 100644 index 0000000000000000000000000000000000000000..96fd8a3bcfdfb56322a6a58c421bf0d9f0cf2188 --- /dev/null +++ b/TRAE-Skills/devops/Chaos_Engineering_Basics.md @@ -0,0 +1,25 @@ +# Skill: Chaos Engineering Basics + +## Purpose +To test system resilience. + +## When to Use +- Simulating failures. +- When the specific requirement for Chaos Engineering Basics arises in the project. + +## Procedure +1. **Analysis**: Understand the requirements for Chaos Engineering Basics. +2. **Implementation**: + - Step 1: Initialize necessary configurations. + - Step 2: Implement the core logic for Chaos Engineering Basics. + - Step 3: Handle edge cases and errors. +3. **Verification**: + - Test the implementation. + - Ensure it meets the project standards. + +## Constraints +- Follow existing project coding standards. +- Ensure performance and security best practices are met. + +## Expected Output +A working implementation of Chaos Engineering Basics with documentation and tests. diff --git a/TRAE-Skills/devops/Cost_Optimization_Cloud.md b/TRAE-Skills/devops/Cost_Optimization_Cloud.md new file mode 100644 index 0000000000000000000000000000000000000000..308ec0bd95497f4e719b34eebe7a0cd6954cc15e --- /dev/null +++ b/TRAE-Skills/devops/Cost_Optimization_Cloud.md @@ -0,0 +1,70 @@ +# Skill: Cost Optimization (AWS/Cloud) + +## Purpose +To reduce cloud infrastructure costs. + +## When to Use +- Spot instances, rightsizing, and budgeting. +- When the specific requirement for Cost Optimization (AWS/Cloud) arises in the project. + +## Procedure + +### 1. Rightsizing Instances +Analyze CPU/Memory utilization and downsize over-provisioned resources. +- Use **AWS Compute Optimizer** or **Azure Advisor** to find underutilized instances. +- Switch to modern instance families (e.g., AWS Graviton/t4g) for better price-performance. + +### 2. Leveraging Spot Instances +Use Spot instances for non-critical, fault-tolerant workloads (CI/CD workers, batch processing). + +```hcl +# Terraform example for Spot Instance +resource "aws_instance" "worker" { + ami = "ami-12345678" + instance_type = "c5.large" + + instance_market_options { + market_type = "spot" + spot_options { + max_price = "0.05" + } + } +} +``` + +### 3. Implementing Auto-Scaling +Ensure infrastructure scales down during low-traffic periods (nights/weekends). +- Set `min_size` to the lowest possible for non-prod environments. +- Use **Scheduled Scaling** for predictable traffic patterns. + +### 4. Storage Optimization +- **Cleanup**: Delete unattached EBS volumes and old S3 versions. +- **Lifecycle Policies**: Move old data to cheaper storage tiers (e.g., S3 Intelligent-Tiering or Glacier). + +```hcl +# S3 Lifecycle Rule +resource "aws_s3_bucket_lifecycle_configuration" "example" { + bucket = aws_s3_bucket.data.id + + rule { + id = "archive_old_logs" + status = "Enabled" + transition { + days = 30 + storage_class = "STANDARD_IA" + } + transition { + days = 90 + storage_class = "GLACIER" + } + } +} +``` + +## Constraints +- **Performance Trade-offs**: Rightsizing too aggressively can lead to performance bottlenecks during peaks. +- **Spot Interruptions**: Always have a fallback strategy for Spot instances (e.g., mixing with On-Demand in an Auto Scaling Group). +- **Data Retention**: Ensure lifecycle policies comply with legal/compliance data retention requirements. + +## Expected Output +A measurable reduction in monthly cloud spend (aim for 20-30%) without compromising application stability. diff --git a/TRAE-Skills/devops/Dependency_Update_Audit.md b/TRAE-Skills/devops/Dependency_Update_Audit.md new file mode 100644 index 0000000000000000000000000000000000000000..f0b2879735eaf95b9d66b6f68decd18c348f8887 --- /dev/null +++ b/TRAE-Skills/devops/Dependency_Update_Audit.md @@ -0,0 +1,29 @@ +# Skill: Dependency Update Audit + +## Purpose +To regularly check, update, and audit project dependencies for security vulnerabilities and deprecations, maintaining a healthy codebase. + +## When to Use +- On a scheduled basis (e.g., monthly). +- When a security advisory is released. +- Before major feature development cycles. + +## Procedure +1. **Check Outdated**: Run `npm outdated` to see available updates. +2. **Audit Security**: Run `npm audit` to identify vulnerabilities. +3. **Update Minor/Patch**: + - Run `npm update` to respect semver ranges in `package.json`. + - Verify app functionality (run tests). +4. **Update Major**: + - Read changelogs for breaking changes. + - Update one major dependency at a time. + - Fix breaking changes and run full test suite. +5. **Lockfile**: Commit the updated `package-lock.json`. + +## Constraints +- Do not blindly update all packages at once, especially major versions. +- Pin exact versions if a package is known to be unstable. +- Fix `npm audit` critical/high vulnerabilities immediately. + +## Expected Output +An updated `package.json` and `package-lock.json` with fewer vulnerabilities and up-to-date libraries. diff --git a/TRAE-Skills/devops/Docker_Containerization_Node.md b/TRAE-Skills/devops/Docker_Containerization_Node.md new file mode 100644 index 0000000000000000000000000000000000000000..54b0dbbab0528f261071d601ee92bf9af4e3c412 --- /dev/null +++ b/TRAE-Skills/devops/Docker_Containerization_Node.md @@ -0,0 +1,100 @@ +# Skill: Docker Containerization (Node.js) + +## Purpose +To package a Node.js application and its dependencies into a lightweight, portable container image that runs consistently across any environment. + +## When to Use +- When preparing an application for production deployment. +- When ensuring development environment consistency. +- When isolating microservices in a cluster (Kubernetes/ECS). + +## Procedure + +### 1. Multi-Stage Dockerfile (TypeScript/Build) +Use multi-stage builds to keep the final production image as small as possible. + +```dockerfile +# Stage 1: Build +FROM node:20-alpine AS builder +WORKDIR /app +COPY package*.json ./ +RUN npm ci +COPY . . +RUN npm run build + +# Stage 2: Production +FROM node:20-alpine +WORKDIR /app +ENV NODE_ENV=production + +# Install only production dependencies +COPY package*.json ./ +RUN npm ci --only=production + +# Copy built assets from builder stage +COPY --from=builder /app/dist ./dist + +# Security: Run as non-root user +USER node + +EXPOSE 3000 +CMD ["node", "dist/index.js"] +``` + +### 2. .dockerignore Configuration +Prevent unnecessary files from bloating the build context. + +```text +node_modules +npm-debug.log +dist +.git +.env +Dockerfile +.dockerignore +``` + +### 3. Docker Compose for Development +Simplify local development with linked services. + +```yaml +# docker-compose.yml +services: + app: + build: + context: . + dockerfile: Dockerfile.dev + ports: + - "3000:3000" + volumes: + - .:/app + - /app/node_modules + environment: + - DATABASE_URL=postgres://user:pass@db:5432/db + depends_on: + - db + + db: + image: postgres:15 + environment: + - POSTGRES_PASSWORD=pass +``` + +### 4. Building and Running +Commands for the CLI. + +```bash +# Build the image +docker build -t my-app:v1 . + +# Run the container +docker run -p 3000:3000 --env-file .env my-app:v1 +``` + +## Constraints +- **Base Image**: Always use `alpine` or `slim` tags for smaller, more secure images. +- **Layer Caching**: Copy `package.json` and install dependencies **before** copying the rest of the source code to leverage Docker layer caching. +- **Security**: Never include `.env` files or hardcoded secrets in the image. Use environment variables at runtime. + +## Expected Output +A production-ready `Dockerfile` that produces a minimal, secure image containing only the necessary runtime files. diff --git a/TRAE-Skills/devops/Docker_Swarm_Orchestration.md b/TRAE-Skills/devops/Docker_Swarm_Orchestration.md new file mode 100644 index 0000000000000000000000000000000000000000..45bafc21e075193f4caaa5974326412b5bcf0223 --- /dev/null +++ b/TRAE-Skills/devops/Docker_Swarm_Orchestration.md @@ -0,0 +1,123 @@ +# Skill: Docker Swarm Orchestration + +## Purpose +To deploy, manage, and scale containerized applications using Docker Swarm for container orchestration. + +## When to Use +- When you need container orchestration but want something simpler than Kubernetes +- For deploying and managing multi-container applications across multiple nodes +- When you need high availability and load balancing for your services +- For rolling updates and rollbacks of application deployments +- When working with a small to medium-sized cluster + +## Procedure + +### 1. Initialize Docker Swarm +Create a Docker Swarm cluster. + +```bash +# On the manager node +docker swarm init --advertise-addr + +# Output will give you a command to join worker nodes +docker swarm join --token :2377 +``` + +### 2. Create a Docker Stack +Deploy services using a docker-compose.yml file. + +```yaml +version: '3.8' + +services: + web: + image: nginx:alpine + ports: + - "80:80" + deploy: + replicas: 3 + update_config: + parallelism: 1 + delay: 10s + restart_policy: + condition: on-failure + networks: + - webnet + + db: + image: postgres:15-alpine + environment: + POSTGRES_PASSWORD: secret + volumes: + - db-data:/var/lib/postgresql/data + deploy: + placement: + constraints: + - node.role == manager + networks: + - webnet + +volumes: + db-data: + +networks: + webnet: + driver: overlay +``` + +### 3. Deploy the Stack +Deploy your application stack. + +```bash +docker stack deploy -c docker-compose.yml myapp +``` + +### 4. Manage Services +Monitor and manage your services. + +```bash +# List stacks +docker stack ls + +# List services in a stack +docker stack services myapp + +# List running containers +docker stack ps myapp + +# Scale a service +docker service scale myapp_web=5 + +# Update a service +docker service update --image nginx:latest myapp_web + +# View service logs +docker service logs myapp_web + +# Remove the stack +docker stack rm myapp +``` + +### 5. Rolling Updates & Rollbacks +Perform rolling updates and rollbacks. + +```bash +# Update the image with rolling update +docker service update \ + --image myapp:v2 \ + --update-parallelism 2 \ + --update-delay 10s \ + myapp_web + +# Rollback if something goes wrong +docker service rollback myapp_web +``` + +## Best Practices +- **Manager Nodes**: Use 3 or 5 manager nodes for high availability +- **Constraints**: Use placement constraints to control where services run +- **Secrets**: Use Docker Secrets for sensitive information instead of environment variables +- **Healthchecks**: Add healthchecks to your services for better reliability +- **Resource Limits**: Set CPU and memory limits for each service +- **Monitoring**: Monitor your swarm with tools like Prometheus and Grafana +- **Backup**: Regularly back up the swarm state (especially manager nodes) diff --git a/TRAE-Skills/devops/Edge_Computing_Vercel_Cloudflare.md b/TRAE-Skills/devops/Edge_Computing_Vercel_Cloudflare.md new file mode 100644 index 0000000000000000000000000000000000000000..a96f5b493b9dd0962d6e07a0ae9d4fc776e44fb5 --- /dev/null +++ b/TRAE-Skills/devops/Edge_Computing_Vercel_Cloudflare.md @@ -0,0 +1,236 @@ +# Skill: Edge Computing with Vercel & Cloudflare Workers + +## Purpose +To build and deploy low-latency, globally distributed edge functions and middleware. + +## When to Use +- When you need ultra-low latency for APIs +- For geolocation-based personalization +- When implementing edge caching and performance optimization +- For A/B testing at the edge +- When handling authentication/authorization at the edge + +## Procedure + +### 1. Vercel Edge Function +Create a simple Vercel Edge Function. + +```typescript +// app/api/edge/route.ts +import { NextResponse } from 'next/server'; + +export const runtime = 'edge'; + +export async function GET(request: Request) { + const { searchParams } = new URL(request.url); + const name = searchParams.get('name') || 'World'; + + // Get geolocation + const geo = request.geo; + const ip = request.headers.get('x-forwarded-for') || 'unknown'; + + return NextResponse.json({ + message: `Hello, ${name}!`, + location: { + city: geo?.city, + country: geo?.country, + region: geo?.region, + }, + ip, + timestamp: new Date().toISOString() + }, { + headers: { + 'Cache-Control': 's-maxage=300, stale-while-revalidate' + } + }); +} +``` + +### 2. Vercel Middleware +Run code before requests reach your app. + +```typescript +// middleware.ts +import { NextResponse } from 'next/server'; +import type { NextRequest } from 'next/server'; + +export function middleware(request: NextRequest) { + // Redirect based on country + const country = request.geo?.country; + if (country === 'CN') { + return NextResponse.redirect(new URL('/cn', request.url)); + } + + // A/B testing + const cookie = request.cookies.get('ab-test'); + let variant = cookie?.value || 'A'; + if (!cookie) { + variant = Math.random() < 0.5 ? 'A' : 'B'; + } + + const response = NextResponse.next(); + response.cookies.set('ab-test', variant, { path: '/' }); + response.headers.set('x-ab-variant', variant); + + // Auth check + const authToken = request.cookies.get('auth-token'); + if (request.nextUrl.pathname.startsWith('/dashboard') && !authToken) { + return NextResponse.redirect(new URL('/login', request.url)); + } + + return response; +} + +export const config = { + matcher: ['/((?!api|_next/static|_next/image|favicon.ico).*)'], +}; +``` + +### 3. Cloudflare Worker +Create a Cloudflare Worker. + +```javascript +// worker.js +export default { + async fetch(request, env, ctx) { + const url = new URL(request.url); + + // Route handling + if (url.pathname === '/api/hello') { + return handleHello(request); + } else if (url.pathname === '/api/cache') { + return handleCache(request, env); + } + + return new Response('Not Found', { status: 404 }); + } +}; + +async function handleHello(request) { + const { cf } = request; + return new Response(JSON.stringify({ + message: 'Hello from Cloudflare Edge!', + colo: cf.colo, + country: cf.country, + city: cf.city, + timezone: cf.timezone + }), { + headers: { 'Content-Type': 'application/json' } + }); +} + +async function handleCache(request, env) { + const cacheKey = request.url; + const cache = caches.default; + + // Check cache + let response = await cache.match(cacheKey); + if (response) { + return new Response(response.body, { + ...response, + headers: { + ...response.headers, + 'x-cache': 'HIT' + } + }); + } + + // Fetch from origin + response = await fetch('https://api.example.com/data'); + const clonedResponse = new Response(response.body, response); + clonedResponse.headers.set('x-cache', 'MISS'); + clonedResponse.headers.set('Cache-Control', 's-maxage=60'); + + // Cache it + ctx.waitUntil(cache.put(cacheKey, clonedResponse.clone())); + + return clonedResponse; +} +``` + +### 4. Cloudflare Worker with KV +Use Cloudflare KV for edge storage. + +```javascript +// worker-kv.js +export default { + async fetch(request, env, ctx) { + const url = new URL(request.url); + + if (url.pathname === '/api/counter') { + // Get current count + let count = await env.COUNTER_KV.get('global-count'); + count = count ? parseInt(count) + 1 : 1; + + // Save to KV + await env.COUNTER_KV.put('global-count', count.toString()); + + return new Response(JSON.stringify({ count }), { + headers: { 'Content-Type': 'application/json' } + }); + } + + if (url.pathname.startsWith('/api/user/')) { + const userId = url.pathname.split('/')[3]; + + if (request.method === 'GET') { + const user = await env.USERS_KV.get(`user:${userId}`); + return new Response(user || '{}', { + headers: { 'Content-Type': 'application/json' } + }); + } + + if (request.method === 'POST') { + const userData = await request.json(); + await env.USERS_KV.put(`user:${userId}`, JSON.stringify(userData)); + return new Response(JSON.stringify({ success: true }), { + headers: { 'Content-Type': 'application/json' } + }); + } + } + + return new Response('Not Found', { status: 404 }); + } +}; +``` + +### 5. Edge Caching with Cloudflare +Implement advanced caching. + +```javascript +// worker-cache.js +export default { + async fetch(request, env) { + const cache = caches.default; + let response = await cache.match(request); + + if (!response) { + // Add cache tags + const cacheTags = ['api', 'data']; + const originResponse = await fetch(request, { + headers: { + 'Cache-Tag': cacheTags.join(',') + } + }); + + response = new Response(originResponse.body, originResponse); + response.headers.set('Cache-Control', 's-maxage=3600, stale-while-revalidate=86400'); + response.headers.set('Cache-Tag', cacheTags.join(',')); + + // Cache the response + ctx.waitUntil(cache.put(request, response.clone())); + } + + return response; + } +}; +``` + +## Best Practices +- **Cold Starts**: Keep edge functions small and fast +- **Caching**: Use edge caching aggressively +- **Geolocation**: Leverage geolocation for personalization +- **KV Storage**: Use KV for edge-side state +- **Cost**: Monitor edge invocation counts +- **Fallback**: Add fallback for edge failures +- **Testing**: Test with wrangler (Cloudflare) or vercel dev diff --git a/TRAE-Skills/devops/Git_Branching_Strategy.md b/TRAE-Skills/devops/Git_Branching_Strategy.md new file mode 100644 index 0000000000000000000000000000000000000000..33e064e9cbf5d979f58876566c82324d28ed5d3f --- /dev/null +++ b/TRAE-Skills/devops/Git_Branching_Strategy.md @@ -0,0 +1,79 @@ +# Skill: Git Branching Strategy (GitFlow/Trunk) + +## Purpose +To standardize how developers collaborate on code, manage releases, and handle hotfixes to avoid conflicts and chaotic history. + +## When to Use +- Setting up a new team workflow. +- Solving "merge hell" situations. + +## Procedure + +### 1. Trunk-Based Development (Recommended for CI/CD) +The most efficient workflow for modern DevOps teams. + +```bash +# 1. Create a short-lived feature branch from main +git checkout main +git pull origin main +git checkout -b feature/user-auth-api + +# 2. Commit frequently with descriptive messages +git add . +git commit -m "feat: add jwt validation middleware" + +# 3. Push and open a Pull Request (PR) +git push origin feature/user-auth-api + +# 4. After PR approval and CI pass, merge to main (Squash merge preferred) +# (Done via GitHub/GitLab UI) + +# 5. Delete local and remote branch +git branch -d feature/user-auth-api +git push origin --delete feature/user-auth-api +``` + +### 2. Standardized Naming Convention +Use prefixes to categorize branches and commits. + +- **`feat/`**: New feature development. +- **`fix/`**: Bug fix for the current environment. +- **`hotfix/`**: Critical fix for production. +- **`chore/`**: Maintenance tasks (updating dependencies, etc.). +- **`docs/`**: Documentation changes only. +- **`refactor/`**: Code change that neither fixes a bug nor adds a feature. + +### 3. Release Management +Use Git Tags to mark production releases. + +```bash +# Create a semantic version tag +git tag -a v1.2.0 -m "Release version 1.2.0: Includes User Auth and Billing fixes" + +# Push tags to remote +git push origin v1.2.0 +``` + +### 4. Handling Conflicts +Rebase or merge from main frequently to minimize conflict size. + +```bash +# Bring latest changes from main into your branch +git checkout feature/my-feature +git fetch origin +git rebase origin/main + +# If conflicts occur: +# 1. Manually fix files in editor +# 2. git add +# 3. git rebase --continue +``` + +## Constraints +- **Branch Longevity**: Feature branches should not live longer than 2-3 days. Long-lived branches lead to complex merge conflicts. +- **Protected Branches**: Enable branch protection on `main` or `develop`. Require at least one approval and passing CI status before merging. +- **Commit History**: Use "Squash and Merge" for PRs to keep the `main` history clean and linear. +- **No Direct Pushes**: Never allow direct pushes to protected branches; everything must go through a PR. + +## Expected Output +A documented workflow policy that the team follows for all code changes. diff --git a/TRAE-Skills/devops/Google_Cloud_Run_Deployment.md b/TRAE-Skills/devops/Google_Cloud_Run_Deployment.md new file mode 100644 index 0000000000000000000000000000000000000000..8f1d6347b45d17e0376b558d5ccd81510e3f5a85 --- /dev/null +++ b/TRAE-Skills/devops/Google_Cloud_Run_Deployment.md @@ -0,0 +1,25 @@ +# Skill: Google Cloud Run Deployment + +## Purpose +To deploy containers serverlessly. + +## When to Use +- Containerizing and deploying to Cloud Run. +- When the specific requirement for Google Cloud Run Deployment arises in the project. + +## Procedure +1. **Analysis**: Understand the requirements for Google Cloud Run Deployment. +2. **Implementation**: + - Step 1: Initialize necessary configurations. + - Step 2: Implement the core logic for Google Cloud Run Deployment. + - Step 3: Handle edge cases and errors. +3. **Verification**: + - Test the implementation. + - Ensure it meets the project standards. + +## Constraints +- Follow existing project coding standards. +- Ensure performance and security best practices are met. + +## Expected Output +A working implementation of Google Cloud Run Deployment with documentation and tests. diff --git a/TRAE-Skills/devops/Infrastructure_as_Code_Terraform.md b/TRAE-Skills/devops/Infrastructure_as_Code_Terraform.md new file mode 100644 index 0000000000000000000000000000000000000000..775d8945b95b7c06778383165be24541594daa04 --- /dev/null +++ b/TRAE-Skills/devops/Infrastructure_as_Code_Terraform.md @@ -0,0 +1,98 @@ +# Skill: Infrastructure as Code (Terraform) + +## Purpose +To provision and manage infrastructure (servers, databases, networks) using declarative configuration files, ensuring reproducibility and version control. + +## When to Use +- Setting up a new cloud environment (AWS, Azure, GCP). +- Managing changes to existing infrastructure. +- Preventing configuration drift. + +## Procedure + +### 1. Provider and Backend Configuration +Define where Terraform will manage resources and store its state. + +```hcl +# main.tf +terraform { + required_version = ">= 1.5.0" + + backend "s3" { + bucket = "my-terraform-state-bucket" + key = "prod/terraform.tfstate" + region = "us-east-1" + dynamodb_table = "terraform-lock-table" + } + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.aws_region +} +``` + +### 2. Resource Definition +Declare your infrastructure components using variables. + +```hcl +# vpc.tf +resource "aws_vpc" "main" { + cidr_block = "10.0.0.0/16" + enable_dns_hostnames = true + + tags = { + Name = "${var.project_name}-vpc" + } +} + +# variables.tf +variable "project_name" { + type = string + default = "my-app" +} +``` + +### 3. Modularization +Create reusable modules for standard components. + +```hcl +# root.tf +module "database" { + source = "./modules/rds" + db_name = "production_db" + instance_class = "db.t3.micro" +} +``` + +### 4. Lifecycle Workflow +Standard commands to manage infrastructure. + +```bash +# 1. Initialize (Downloads providers/modules) +terraform init + +# 2. Plan (Shows what will happen) +terraform plan -out=tfplan + +# 3. Apply (Executes changes) +terraform apply "tfplan" + +# 4. Format/Validate +terraform fmt -recursive +terraform validate +``` + +## Constraints +- **State Security**: Never commit `.tfstate` to Git. Use remote backends with encryption. +- **Destructive Changes**: Always review `terraform plan` output carefully, especially for "replace" actions on databases. +- **Variables**: Use `terraform.tfvars` or environment variables for sensitive inputs. + +## Expected Output +A modular, versioned infrastructure definition that can recreate environments consistently with a single command. diff --git a/TRAE-Skills/devops/Kubernetes_Deployment_Manifests.md b/TRAE-Skills/devops/Kubernetes_Deployment_Manifests.md new file mode 100644 index 0000000000000000000000000000000000000000..9e641e7eab0877d2fcf7eba46a58596813b2c9d7 --- /dev/null +++ b/TRAE-Skills/devops/Kubernetes_Deployment_Manifests.md @@ -0,0 +1,129 @@ +# Skill: Kubernetes Deployment Manifests + +## Purpose +To define how applications should run, scale, and update within a Kubernetes cluster using declarative YAML manifests. + +## When to Use +- Deploying containerized applications to K8s. +- Defining resource limits, replicas, and environment variables. + +## Procedure + +### 1. Deployment Manifest +Create a `deployment.yaml` to manage your application's lifecycle. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-app-deployment + labels: + app: my-app +spec: + replicas: 3 + selector: + matchLabels: + app: my-app + template: + metadata: + labels: + app: my-app + spec: + containers: + - name: my-app + image: my-registry/my-app:1.2.3 + ports: + - containerPort: 3000 + envFrom: + - configMapRef: + name: my-app-config + - secretRef: + name: my-app-secrets + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: /health + port: 3000 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /ready + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 +``` + +### 2. Service Manifest +Create a `service.yaml` to expose the application within the cluster. + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: my-app-service +spec: + selector: + app: my-app + ports: + - protocol: TCP + port: 80 + targetPort: 3000 + type: ClusterIP # Internal exposure only +``` + +### 3. ConfigMap and Secrets +Decouple configuration from the deployment. + +```yaml +# configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-app-config +data: + NODE_ENV: "production" + LOG_LEVEL: "info" + +--- +# secret.yaml (values must be base64 encoded) +apiVersion: v1 +kind: Secret +metadata: + name: my-app-secrets +type: Opaque +data: + DATABASE_URL: bXktZGItaG9zdC1jb25uZWN0aW9uLXN0cmluZw== +``` + +### 4. Applying and Managing +Use `kubectl` to apply the manifests and check status. + +```bash +# Apply all files in the directory +kubectl apply -f ./k8s/ + +# Check rollout status +kubectl rollout status deployment/my-app-deployment + +# Get logs from a specific pod +kubectl logs -l app=my-app --tail=100 + +# Scale the deployment +kubectl scale deployment/my-app-deployment --replicas=5 +``` + +## Constraints +- **Immutable Tags**: Do not use the `latest` tag; use specific versions or image SHAs to ensure reproducibility. +- **Resource Limits**: Always set CPU and Memory requests and limits to prevent a single container from crashing the node. +- **Probes**: Implement both `liveness` (restart if hung) and `readiness` (don't send traffic until ready) probes. +- **Secrets Management**: Do not commit raw `secret.yaml` files to Git. Use tools like Sealed Secrets, External Secrets, or HashiCorp Vault. + +## Expected Output +Valid YAML files (`deployment.yaml`, `service.yaml`) that successfully launch the application in a cluster. diff --git a/TRAE-Skills/devops/Kubernetes_Helm_Charts.md b/TRAE-Skills/devops/Kubernetes_Helm_Charts.md new file mode 100644 index 0000000000000000000000000000000000000000..c9a0ac5ea6015e542cdf85fd725814707e805410 --- /dev/null +++ b/TRAE-Skills/devops/Kubernetes_Helm_Charts.md @@ -0,0 +1,276 @@ +# Skill: Kubernetes Helm Charts + +## Purpose +To package, deploy, and manage Kubernetes applications using Helm charts. + +## When to Use +- When deploying complex applications to Kubernetes +- For managing multiple environments (dev, staging, prod) +- When sharing applications with others +- For versioning and rolling back application deployments +- When you need reusable application templates + +## Procedure + +### 1. Create a Helm Chart +Scaffold a new Helm chart. + +```bash +# Create a new chart +helm create myapp + +# Explore the chart structure +cd myapp +ls -la +``` + +### 2. Chart Structure +Understand the Helm chart structure. + +``` +myapp/ +├── Chart.yaml # Chart metadata +├── values.yaml # Default configuration values +├── charts/ # Dependency charts +├── templates/ # Kubernetes manifest templates +│ ├── NOTES.txt # Usage notes +│ ├── _helpers.tpl # Template helpers +│ ├── deployment.yaml +│ ├── service.yaml +│ ├── ingress.yaml +│ └── tests/ +└── .helmignore # Files to ignore +``` + +### 3. Chart.yaml +Define chart metadata. + +```yaml +apiVersion: v2 +name: myapp +description: A Helm chart for my application +type: application +version: 0.1.0 +appVersion: "1.0.0" +keywords: + - myapp + - web +maintainers: + - name: Your Name + email: your.email@example.com +``` + +### 4. values.yaml +Define default configuration values. + +```yaml +replicaCount: 3 + +image: + repository: nginx + pullPolicy: IfNotPresent + tag: "1.25.0" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: "" + +podAnnotations: {} +podSecurityContext: {} + +securityContext: {} + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} +``` + +### 5. Template Deployment +Create Kubernetes manifest templates. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "myapp.fullname" . }} + labels: + {{- include "myapp.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "myapp.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "myapp.labels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "myapp.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +``` + +### 6. Deploy the Chart +Install and manage the chart. + +```bash +# Install the chart +helm install myapp ./myapp + +# Install with custom values +helm install myapp ./myapp -f values-production.yaml + +# Install with set values +helm install myapp ./myapp --set replicaCount=5,image.tag=1.26.0 + +# List releases +helm list + +# Upgrade the release +helm upgrade myapp ./myapp --set image.tag=1.27.0 + +# Rollback to previous version +helm rollback myapp + +# Uninstall the release +helm uninstall myapp + +# Template the chart (render manifests without installing) +helm template myapp ./myapp + +# Lint the chart +helm lint ./myapp +``` + +### 7. Environment-Specific Values +Create values files for different environments. + +```yaml +# values-dev.yaml +replicaCount: 1 +ingress: + enabled: true + hosts: + - host: dev.myapp.com + paths: + - path: / + pathType: ImplementationSpecific + +# values-staging.yaml +replicaCount: 2 +ingress: + enabled: true + hosts: + - host: staging.myapp.com + paths: + - path: / + pathType: ImplementationSpecific + +# values-prod.yaml +replicaCount: 5 +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 250m + memory: 256Mi +autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 +ingress: + enabled: true + hosts: + - host: myapp.com + paths: + - path: / + pathType: ImplementationSpecific +``` + +## Best Practices +- **Semantic Versioning**: Follow semantic versioning for charts +- **Default Values**: Provide sensible defaults in values.yaml +- **Documentation**: Document all values in values.yaml comments +- **Template Helpers**: Use _helpers.tpl for reusable template functions +- **Testing**: Write chart tests in the templates/tests directory +- **Dependencies**: Declare dependencies in Chart.yaml +- **Security**: Use least-privilege security contexts +- **Linting**: Always lint charts before deployment diff --git a/TRAE-Skills/devops/Log_Aggregation_ELK.md b/TRAE-Skills/devops/Log_Aggregation_ELK.md new file mode 100644 index 0000000000000000000000000000000000000000..400e25cd7e21c8eb9760ea8b51f4cbeee30424da --- /dev/null +++ b/TRAE-Skills/devops/Log_Aggregation_ELK.md @@ -0,0 +1,89 @@ +# Skill: Log Aggregation (ELK/Loki) + +## Purpose +To centralize logs from all services and containers into a single queryable interface, enabling debugging and trend analysis across the distributed system. + +## When to Use +- When running multiple microservices or containers. +- When SSH-ing into servers to check logs is not scalable. + +## Procedure + +### 1. Application Logging (Node.js with Winston) +Configure your application to output structured logs (JSON) to standard output. + +```javascript +const winston = require('winston'); + +const logger = winston.createLogger({ + level: 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json() // Essential for log aggregators to parse easily + ), + defaultMeta: { service: 'user-service' }, + transports: [ + new winston.transports.Console(), + ], +}); + +// Example log +logger.info('User logged in', { userId: '123', ip: '192.168.1.1' }); +``` + +### 2. Log Collection (Filebeat for ELK) +Configure Filebeat to harvest logs from Docker containers. + +```yaml +# filebeat.yml +filebeat.inputs: +- type: container + paths: + - /var/lib/docker/containers/*/*.log + +processors: +- add_docker_metadata: ~ +- decode_json_fields: + fields: ["message"] + target: "json" + overwrite_keys: true + +output.elasticsearch: + hosts: ["elasticsearch:9200"] + index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}" +``` + +### 3. Log Shipping (Promtail for Loki) +If using the PLG stack, configure Promtail as a DaemonSet. + +```yaml +# promtail-config.yaml +server: + http_listen_port: 9080 + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: +- job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app] + target_label: app +``` + +### 4. Querying and Visualization +- **Kibana (ELK)**: Use KQL (Kibana Query Language) to search logs. + - `json.userId: "123" AND level: "error"` +- **Grafana (Loki)**: Use LogQL to filter and aggregate. + - `{app="user-service"} |= "error" | json` + +## Constraints +- **Structured Logging**: Always log in JSON format. Parsing unstructured text logs with regex is brittle and CPU-intensive for the aggregator. +- **Log Rotation**: Ensure the host system rotates local log files (e.g., via Docker's `max-size` and `max-file` options) to prevent disk exhaustion. +- **PII Redaction**: Never log sensitive information like passwords, credit card numbers, or full PII. Implement redaction in the logging library or the shipping agent. +- **Volume Management**: Use sampling or level filtering (e.g., only `warn` and `error` in production) if log volume becomes too expensive. + +## Expected Output +A logging pipeline where application logs appear in the visualization tool within seconds of generation. diff --git a/TRAE-Skills/devops/Monitoring_with_Datadog.md b/TRAE-Skills/devops/Monitoring_with_Datadog.md new file mode 100644 index 0000000000000000000000000000000000000000..85d4b34f83ae8fd1bee638f5753e0ed99c93fec1 --- /dev/null +++ b/TRAE-Skills/devops/Monitoring_with_Datadog.md @@ -0,0 +1,77 @@ +# Skill: Monitoring with Datadog + +## Purpose +To observe system health. + +## When to Use +- Metrics, dashboards, and alerts. +- When the specific requirement for Monitoring with Datadog arises in the project. + +## Procedure + +### 1. Agent Installation +Deploy the Datadog Agent to your infrastructure (using Docker/Kubernetes). + +```yaml +# datadog-agent.yaml (K8s) +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: datadog-agent +spec: + template: + spec: + containers: + - name: datadog-agent + image: gcr.io/datadoghq/agent:latest + env: + - name: DD_API_KEY + valueFrom: + secretKeyRef: + name: datadog-secret + key: api-key +``` + +### 2. Application Instrumentation (Node.js) +Install the `dd-trace` library and initialize it at the very top of your entry file. + +```javascript +// index.js - MUST BE THE FIRST LINE +require('dd-trace').init({ + env: 'production', + service: 'my-backend-service', + version: '1.0.0', + logInjection: true // Links logs to traces +}); + +const express = require('express'); +const app = express(); +// ... rest of your app +``` + +### 3. Custom Metrics and Tags +Send custom business metrics to Datadog. + +```javascript +const stats = require('hot-shots'); +const dogstatsd = new stats(); + +// Increment a counter with tags +dogstatsd.increment('checkout.completed', 1, ['plan:pro', 'region:us-east']); + +// Gauge for current status +dogstatsd.gauge('active.sessions', 150); +``` + +### 4. Setting up Monitors (Alerts) +Create monitors via the UI or Terraform to alert on specific thresholds. +- **Metric Monitor**: Alert if `avg(last_5m):avg:system.cpu.user{host:my-app} > 80`. +- **Anomaly Detection**: Alert if request latency deviates from seasonal patterns. + +## Constraints +- **Sampling**: Adjust sampling rates for high-traffic apps to control costs. +- **Tagging**: Use consistent tags (`env`, `service`, `team`) for efficient filtering. +- **Security**: Mask PII (Personally Identifiable Information) in logs and traces using Datadog's sensitive data scanner. + +## Expected Output +A fully instrumented application providing real-time dashboards, traces, and automated alerts in the Datadog platform. diff --git a/TRAE-Skills/devops/Nginx_Reverse_Proxy_Setup.md b/TRAE-Skills/devops/Nginx_Reverse_Proxy_Setup.md new file mode 100644 index 0000000000000000000000000000000000000000..88cd29368b40d961d60639debe826d6537b77133 --- /dev/null +++ b/TRAE-Skills/devops/Nginx_Reverse_Proxy_Setup.md @@ -0,0 +1,101 @@ +# Skill: Nginx Reverse Proxy Setup + +## Purpose +To sit in front of backend applications, handling SSL termination, load balancing, compression, and static file serving. + +## When to Use +- Exposing a Node.js app to the internet. +- Hosting multiple domains on one server. +- Improving security and performance. + +## Procedure + +### 1. Basic Server Block Configuration +Create a configuration file in `/etc/nginx/sites-available/myapp.conf`. + +```nginx +upstream myapp_upstream { + server 127.0.0.1:3000; + keepalive 64; +} + +server { + listen 80; + server_name example.com www.example.com; + + # Redirect all HTTP traffic to HTTPS + return 301 https://$server_name$request_uri; +} + +server { + listen 443 ssl http2; + server_name example.com www.example.com; + + # SSL Configuration (assuming Certbot paths) + ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; + + # Performance & Security + client_max_body_size 20M; + server_tokens off; + + location / { + proxy_pass http://myapp_upstream; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Timeouts + proxy_connect_timeout 60s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + } + + # Static Assets Caching + location /static/ { + alias /var/www/myapp/static/; + expires 30d; + add_header Cache-Control "public, no-transform"; + } +} +``` + +### 2. Enabling the Configuration +Link the file to the enabled sites directory and test the configuration. + +```bash +# Enable the site +ln -s /etc/nginx/sites-available/myapp.conf /etc/nginx/sites-enabled/ + +# Test configuration syntax +nginx -t + +# Reload Nginx to apply changes +systemctl reload nginx +``` + +### 3. Adding Security Headers +Include common security headers in the `server` block or a separate snippet. + +```nginx +add_header X-Frame-Options "SAMEORIGIN" always; +add_header X-XSS-Protection "1; mode=block" always; +add_header X-Content-Type-Options "nosniff" always; +add_header Referrer-Policy "no-referrer-when-downgrade" always; +add_header Content-Security-Policy "default-src 'self' http: https: data: blob: 'unsafe-inline'" always; +add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; +``` + +## Constraints +- **IP Forwarding**: Always pass the real client IP (`X-Forwarded-For`) to ensure backend logs and rate limiting work correctly. +- **Header Limits**: Be careful with `proxy_buffer_size` if your app sends large headers (e.g., heavy JWTs). +- **Security**: Disable server tokens (`server_tokens off;`) to avoid leaking the Nginx version to attackers. +- **Permissions**: Ensure the `nginx` user has read access to static asset directories. + +## Expected Output +A configured Nginx server routing traffic to the application, handling HTTP/HTTPS correctly. diff --git a/TRAE-Skills/devops/Prometheus_Grafana_Monitoring.md b/TRAE-Skills/devops/Prometheus_Grafana_Monitoring.md new file mode 100644 index 0000000000000000000000000000000000000000..15ceeefd809fc34c15c800d404c2260e91fa6862 --- /dev/null +++ b/TRAE-Skills/devops/Prometheus_Grafana_Monitoring.md @@ -0,0 +1,86 @@ +# Skill: Monitoring & Alerting (Prometheus/Grafana) + +## Purpose +To collect metrics (CPU, Memory, Request Latency, Error Rates) and alert engineers when the system behaves abnormally. + +## When to Use +- Production readiness. +- Capacity planning. +- Detecting incidents before users report them. + +## Procedure + +### 1. Application Instrumentation (Node.js) +Install `prom-client` and expose the `/metrics` endpoint. + +```javascript +const express = require('express'); +const client = require('prom-client'); + +const app = express(); +const register = new client.Registry(); + +// Add default metrics (CPU, Memory, etc.) +client.collectDefaultMetrics({ register }); + +// Define a custom metric +const httpRequestDuration = new client.Histogram({ + name: 'http_request_duration_seconds', + help: 'Duration of HTTP requests in seconds', + labelNames: ['method', 'route', 'status_code'], + buckets: [0.1, 0.5, 1, 2, 5] +}); +register.registerMetric(httpRequestDuration); + +app.get('/metrics', async (req, res) => { + res.set('Content-Type', register.contentType); + res.end(await register.metrics()); +}); +``` + +### 2. Prometheus Configuration +Configure Prometheus to scrape your application in `prometheus.yml`. + +```yaml +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'my-backend-service' + static_configs: + - targets: ['myapp:3000'] + metrics_path: '/metrics' +``` + +### 3. Alerting Rules +Define rules for Alertmanager to trigger notifications. + +```yaml +# alert_rules.yml +groups: +- name: APIAlerts + rules: + - alert: HighErrorRate + expr: rate(http_requests_total{status_code=~"5.."}[5m]) / rate(http_requests_total[5m]) > 0.05 + for: 2m + labels: + severity: critical + annotations: + summary: "High Error Rate on {{ $labels.instance }}" + description: "Error rate is above 5% for more than 2 minutes." +``` + +### 4. Grafana Dashboard Setup +1. Add Prometheus as a Data Source. +2. Import common dashboards (e.g., ID: 1860 for Node.js Exporter). +3. Create custom panels using PromQL: + - **Throughput**: `sum(rate(http_requests_total[1m])) by (method)` + - **P95 Latency**: `histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))` + +## Constraints +- **Cardinality**: Never use high-cardinality values (like User IDs, UUIDs, or email addresses) as Prometheus labels. This will cause memory usage to explode. +- **Actionable Alerts**: Only alert on symptoms that affect users (e.g., high error rate, high latency) rather than causes (e.g., high CPU) unless they are predictive of failure. +- **Scrape Interval**: Ensure the `scrape_interval` is consistent with your alerting `for` duration to avoid false positives. + +## Expected Output +A monitoring dashboard and an active alerting pipeline for critical system health indicators. diff --git a/TRAE-Skills/devops/SSL_TLS_Certbot_Setup.md b/TRAE-Skills/devops/SSL_TLS_Certbot_Setup.md new file mode 100644 index 0000000000000000000000000000000000000000..210eac57e4c5cc10dd7ab962995e9df3b87cba5b --- /dev/null +++ b/TRAE-Skills/devops/SSL_TLS_Certbot_Setup.md @@ -0,0 +1,75 @@ +# Skill: SSL/TLS Certificate Management (Certbot) + +## Purpose +To secure communication between clients and the server using HTTPS, obtaining free certificates from Let's Encrypt and automating renewal. + +## When to Use +- Any public-facing web service. +- Compliance requirements (PCI-DSS). + +## Procedure + +### 1. Installation of Certbot and Nginx Plugin +Install the necessary packages for your OS (Ubuntu/Debian example). + +```bash +sudo apt update +sudo apt install certbot python3-certbot-nginx +``` + +### 2. Obtaining and Installing the Certificate +Run Certbot to automatically fetch the certificate and configure Nginx. + +```bash +# Request certificate for a single domain +sudo certbot --nginx -d example.com -d www.example.com + +# Follow the prompts: +# 1. Enter email for renewal notifications +# 2. Agree to Terms of Service +# 3. Choose whether to redirect HTTP to HTTPS (Recommended: Yes) +``` + +### 3. Manual Certificate Generation (DNS Challenge) +If you are using a wildcard certificate or don't have port 80 open. + +```bash +sudo certbot certonly --manual --preferred-challenges dns -d "*.example.com" -d "example.com" + +# You will need to add a TXT record to your DNS provider as instructed +``` + +### 4. Verifying and Testing Auto-Renewal +Certbot usually adds a systemd timer or cron job automatically. + +```bash +# Test the renewal process without making changes +sudo certbot renew --dry-run + +# Check systemd timer status +systemctl status certbot.timer +``` + +### 5. Nginx Configuration Check +Ensure the generated configuration in `/etc/nginx/sites-available/` looks correct. + +```nginx +server { + listen 443 ssl; # managed by Certbot + server_name example.com; + + ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot +} +``` + +## Constraints +- **Domain Resolution**: The domain must point to the server's public IP before running Certbot. +- **Port 80**: Must be open and not used by another process (unless using the `--nginx` plugin which handles this). +- **Rate Limits**: Let's Encrypt has strict limits (e.g., 50 certificates per registered domain per week). +- **Security**: Protect the `/etc/letsencrypt/` directory; it contains your private keys. + +## Expected Output +A secured website with a valid, auto-renewing SSL certificate. diff --git a/TRAE-Skills/devops/Sentry_Error_Tracking.md b/TRAE-Skills/devops/Sentry_Error_Tracking.md new file mode 100644 index 0000000000000000000000000000000000000000..597c7c5d3e33bab97e9cfb563243d446f69a82c8 --- /dev/null +++ b/TRAE-Skills/devops/Sentry_Error_Tracking.md @@ -0,0 +1,74 @@ +# Skill: Sentry Error Tracking Setup + +## Purpose +To track application errors. + +## When to Use +- SDK setup and source maps. +- When the specific requirement for Sentry Error Tracking Setup arises in the project. + +## Procedure + +### 1. SDK Installation +Install the Sentry SDK for your platform. + +```bash +# For Node.js +npm install @sentry/node @sentry/profiling-node +``` + +### 2. Initialization +Initialize Sentry as early as possible in your application. + +```javascript +import * as Sentry from "@sentry/node"; +import { nodeProfilingIntegration } from "@sentry/profiling-node"; + +Sentry.init({ + dsn: "https://your-public-key@sentry.io/project-id", + integrations: [ + nodeProfilingIntegration(), + ], + // Performance Monitoring + tracesSampleRate: 1.0, // Capture 100% of transactions in dev + // Set sampling rate for profiling - this is relative to tracesSampleRate + profilesSampleRate: 1.0, +}); +``` + +### 3. Capturing Errors +Sentry automatically captures unhandled exceptions, but you can also capture errors manually. + +```javascript +try { + doSomethingRisky(); +} catch (e) { + Sentry.captureException(e, { + extra: { context: "risky_operation" }, + tags: { feature: "billing" } + }); +} +``` + +### 4. Integrating with Express +Add Sentry handlers to your Express app for automatic request tracking. + +```javascript +const app = express(); + +// The request handler must be the first middleware on the app +app.use(Sentry.Handlers.requestHandler()); + +// ... your routes ... + +// The error handler must be before any other error middleware and after all controllers +app.use(Sentry.Handlers.errorHandler()); +``` + +## Constraints +- **PII Protection**: Use `beforeSend` or Sentry's data scrubbing features to remove sensitive user data (passwords, tokens) from reports. +- **Source Maps**: Upload source maps to Sentry during your build process to see original code in stack traces instead of minified code. +- **Sampling**: In high-traffic production environments, lower the `tracesSampleRate` (e.g., 0.1 or 0.01) to avoid performance impact and cost. + +## Expected Output +Real-time error notifications with full stack traces, breadcrumbs (user actions leading to the error), and environment details in the Sentry dashboard. diff --git a/TRAE-Skills/devops/Serverless_Framework_Setup.md b/TRAE-Skills/devops/Serverless_Framework_Setup.md new file mode 100644 index 0000000000000000000000000000000000000000..1f08da31d8abf7878e9a285b8a1a4cf48c14593c --- /dev/null +++ b/TRAE-Skills/devops/Serverless_Framework_Setup.md @@ -0,0 +1,76 @@ +# Skill: Serverless Framework Setup + +## Purpose +To deploy serverless apps easily. + +## When to Use +- serverless.yml configuration. +- When the specific requirement for Serverless Framework Setup arises in the project. + +## Procedure + +### 1. Installation and Configuration +Install the Serverless CLI and configure your cloud provider credentials. + +```bash +npm install -g serverless + +# Configure AWS credentials +serverless config credentials --provider aws --key --secret +``` + +### 2. Creating a Service +Initialize a new project from a template. + +```bash +serverless create --template aws-nodejs-typescript --path my-service +cd my-service +npm install +``` + +### 3. Configuring `serverless.yml` +Define your infrastructure and functions in the configuration file. + +```yaml +service: my-backend + +provider: + name: aws + runtime: nodejs18.x + region: us-east-1 + stage: ${opt:stage, 'dev'} + environment: + DATABASE_URL: ${env:DATABASE_URL} + +functions: + createUser: + handler: handler.createUser + events: + - http: + path: users + method: post + cors: true + +plugins: + - serverless-offline # For local development + - serverless-esbuild # For fast bundling +``` + +### 4. Deployment +Deploy your service to the cloud. + +```bash +# Deploy to the default stage (dev) +serverless deploy + +# Deploy to production +serverless deploy --stage prod +``` + +## Constraints +- **Function Size**: Keep function bundles small (use `esbuild` or `webpack`) to minimize cold start times. +- **Permissions**: Use the principle of least privilege for IAM roles assigned to functions. +- **VPC Latency**: Be careful when placing Lambda functions inside a VPC, as it can increase cold start times (though improved in recent years). + +## Expected Output +A set of cloud resources (Lambda, API Gateway, IAM roles) deployed and managed via the Serverless Framework. diff --git a/TRAE-Skills/devops/Terraform_Advanced.md b/TRAE-Skills/devops/Terraform_Advanced.md new file mode 100644 index 0000000000000000000000000000000000000000..fda4a6ab85a2d6253db919421ca370b3b885e5a5 --- /dev/null +++ b/TRAE-Skills/devops/Terraform_Advanced.md @@ -0,0 +1,25 @@ +# Skill: Terraform Best Practices (Advanced) + +## Purpose +To manage complex infrastructure. + +## When to Use +- Modules, state management, and workspaces. +- When the specific requirement for Terraform Best Practices (Advanced) arises in the project. + +## Procedure +1. **Analysis**: Understand the requirements for Terraform Best Practices (Advanced). +2. **Implementation**: + - Step 1: Initialize necessary configurations. + - Step 2: Implement the core logic for Terraform Best Practices (Advanced). + - Step 3: Handle edge cases and errors. +3. **Verification**: + - Test the implementation. + - Ensure it meets the project standards. + +## Constraints +- Follow existing project coding standards. +- Ensure performance and security best practices are met. + +## Expected Output +A working implementation of Terraform Best Practices (Advanced) with documentation and tests. diff --git a/TRAE-Skills/documentation/API_Design_Guidelines_REST.md b/TRAE-Skills/documentation/API_Design_Guidelines_REST.md new file mode 100644 index 0000000000000000000000000000000000000000..ec4b6147ed96f64f0bb5b0a853c79fc52f0278b0 --- /dev/null +++ b/TRAE-Skills/documentation/API_Design_Guidelines_REST.md @@ -0,0 +1,72 @@ +# Skill: API Design Guidelines (REST/JSON) + +## Purpose +To establish a consistent, intuitive, and future-proof set of rules for designing RESTful APIs. Good API design improves developer experience (DX), simplifies maintenance, and ensures consistency across microservices. + +## When to Use +- When designing new API endpoints or versioning existing ones +- When establishing an API-first development process +- When creating documentation for external consumers +- When building backend services that communicate with multiple frontends (Mobile, Web, IoT) + +## Procedure + +### 1. Resource Naming (URIs) +Resources should be **nouns**, never verbs. Use plural nouns for consistency. + +**❌ BAD**: `GET /getUsers`, `POST /createUser` +**✅ GOOD**: `GET /users`, `POST /users` + +**Resource Hierarchy**: +`GET /users/123/orders/456` (Order 456 belonging to User 123) + +### 2. HTTP Methods (Verbs) +Use HTTP verbs to define the action. + +- `GET`: Retrieve a resource or collection. (Idempotent, safe) +- `POST`: Create a new resource. (Non-idempotent) +- `PUT`: Replace a resource entirely. (Idempotent) +- `PATCH`: Update a resource partially. (Non-idempotent) +- `DELETE`: Remove a resource. (Idempotent) + +### 3. Status Codes +Use standard HTTP status codes to communicate the result. + +- `200 OK`: Successful request. +- `201 Created`: Successful resource creation. +- `204 No Content`: Successful request, but no response body (e.g., after DELETE). +- `400 Bad Request`: Client-side error (invalid input). +- `401 Unauthorized`: Missing or invalid authentication. +- `403 Forbidden`: Authenticated, but no permission for this resource. +- `404 Not Found`: Resource does not exist. +- `429 Too Many Requests`: Rate limit exceeded. +- `500 Internal Server Error`: Server-side crash or unexpected error. + +### 4. Query Parameters (Filtering, Sorting, Pagination) +Don't include filters in the path. Use query parameters instead. + +- **Filtering**: `GET /users?status=active` +- **Sorting**: `GET /users?sort=-created_at` (Prefix with `-` for descending) +- **Pagination**: `GET /users?page=2&limit=50` (or `offset=50&limit=50`) + +### 5. Error Response Format +Always return a consistent JSON structure for errors. + +```json +{ + "error": { + "code": "VALIDATION_FAILED", + "message": "Invalid email address format", + "details": [ + { "field": "email", "issue": "must be a valid email" } + ] + } +} +``` + +## Best Practices +- **Version Your API**: Use URI versioning (e.g., `/v1/users`) or header versioning to avoid breaking existing clients when making major changes. +- **Use JSON Only**: Set the `Content-Type: application/json` header and always return JSON objects (not arrays) at the top level for future extensibility. +- **Use HATEOAS (Optional but Good)**: Provide links to related resources in the response body to make the API discoverable. +- **CamelCase vs snake_case**: Choose one for your keys and be consistent. `snake_case` is common in many public APIs (GitHub, Stripe). +- **Security First**: Never include sensitive data (passwords, internal IDs, debug info) in the response body. \ No newline at end of file diff --git a/TRAE-Skills/documentation/API_Documentation_Best_Practices.md b/TRAE-Skills/documentation/API_Documentation_Best_Practices.md new file mode 100644 index 0000000000000000000000000000000000000000..e4db8e8a5b83a15976a3b0d531a15ee841235873 --- /dev/null +++ b/TRAE-Skills/documentation/API_Documentation_Best_Practices.md @@ -0,0 +1,15 @@ +# API Documentation Best Practices + +## Overview +Good API documentation is essential for developer experience and adoption. + +## Tools +- **Swagger/OpenAPI**: Standard for REST APIs. Auto-generate from code where possible. +- **Postman Collections**: Shareable request collections. +- **GraphiQL/Apollo Studio**: For GraphQL APIs. + +## Content +- **Authentication**: How to authenticate (Bearer, API Key). +- **Endpoints**: Method, URL, Parameters. +- **Request/Response Examples**: JSON bodies for success and error cases. +- **Error Codes**: Explain standard error responses. diff --git a/TRAE-Skills/documentation/Architectural_Decision_Records_ADR.md b/TRAE-Skills/documentation/Architectural_Decision_Records_ADR.md new file mode 100644 index 0000000000000000000000000000000000000000..f62934bc597e22b09dd99d2992bf363300ac20bb --- /dev/null +++ b/TRAE-Skills/documentation/Architectural_Decision_Records_ADR.md @@ -0,0 +1,56 @@ +# Skill: Architectural Decision Records (ADR) + +## Purpose +To document the "Why" behind significant architectural decisions in a way that is easily searchable and immutable over time. ADRs provide context for future developers about the constraints, trade-offs, and rationale that led to a specific system design, preventing "architectural amnesia." + +## When to Use +- When choosing a primary database (e.g., PostgreSQL vs. MongoDB) +- When adopting a new framework or major library (e.g., React vs. Vue, Next.js) +- When defining a communication protocol between services (e.g., REST, GraphQL, gRPC) +- When implementing a core design pattern (e.g., CQRS, Event Sourcing) + +## Procedure + +### 1. The Structure of an ADR +An ADR should be a short, focused Markdown file (usually stored in a `/docs/adr/` folder). + +**ADR Template**: +- **Title**: Short and descriptive (e.g., `ADR 001: Use PostgreSQL for User Data`) +- **Date**: YYYY-MM-DD +- **Status**: `Proposed`, `Accepted`, `Deprecated`, or `Superseded` (by ADR XXX) +- **Context**: What is the problem we are solving? What are the constraints? +- **Decision**: What did we decide to do? +- **Consequences**: What are the trade-offs? What is the impact (positive and negative)? + +### 2. Format Example + +```markdown +# ADR 005: Use GraphQL for Frontend-Backend Communication + +**Status**: Accepted +**Date**: 2023-10-27 + +## Context +Our current REST API requires multiple round-trips to fetch data for the dashboard, leading to high latency on mobile devices. Frontend developers are frequently blocked waiting for new REST endpoints to be created. + +## Decision +We will use GraphQL (Apollo Server/Client) as the primary communication layer between our web/mobile apps and our microservices. + +## Consequences +- **Positive**: Frontend can query exactly what they need, reducing payload size. +- **Positive**: Schema introspection provides better documentation and type safety. +- **Negative**: Increased complexity in handling caching (N+1 problem). +- **Negative**: Steep learning curve for the team compared to REST. +``` + +### 3. Managing ADRs +- **Immutable**: Once an ADR is `Accepted`, it should not be edited. If the decision changes later, create a *new* ADR that `Supersedes` the old one. +- **Sequential Numbering**: Use a 3-digit prefix (e.g., `001`, `002`) to maintain order. +- **Git Flow**: ADRs should be submitted as Pull Requests. This allows the team to discuss and refine the decision before it's officially accepted. + +## Best Practices +- **Focus on the "Why"**: Don't just document the decision; document the *alternatives* that were rejected and the specific reasons why. +- **Keep it Short**: An ADR should be readable in 5 minutes. If it's too long, it might be a Technical Spec instead. +- **One Decision Per ADR**: Don't bundle unrelated decisions (e.g., "Use React and PostgreSQL") into one file. +- **Visible to All**: Store ADRs in the main repository so every developer can find them. +- **Refer to ADRs in PRs**: When a PR implements a major decision, link to the relevant ADR in the description. \ No newline at end of file diff --git a/TRAE-Skills/documentation/Changelog_Maintenance.md b/TRAE-Skills/documentation/Changelog_Maintenance.md new file mode 100644 index 0000000000000000000000000000000000000000..26f1650f4ca81cd3492dabf5005e9ec4d4b1f723 --- /dev/null +++ b/TRAE-Skills/documentation/Changelog_Maintenance.md @@ -0,0 +1,71 @@ +# Skill: Release Notes & Changelog Maintenance + +## Purpose +To communicate product updates, bug fixes, and security patches to users and internal stakeholders in a clear, consistent, and useful format. A good changelog builds trust and helps users understand how the product is evolving. + +## When to Use +- After every production deployment or version release +- When publishing open-source libraries (e.g., via npm, PyPI) +- To maintain a historical record of system changes for audit or debugging purposes + +## Procedure + +### 1. The Standard: Keep a Changelog +Follow the [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. + +**`CHANGELOG.md` Structure**: +- `Added`: For new features. +- `Changed`: For changes in existing functionality. +- `Deprecated`: For soon-to-be-removed features. +- `Removed`: For now-removed features. +- `Fixed`: For any bug fixes. +- `Security`: In case of vulnerabilities. + +### 2. Format Example + +```markdown +# Changelog + +All notable changes to this project will be documented in this file. + +## [1.2.0] - 2023-10-27 +### Added +- Multi-language support (English, Italian, Spanish). +- Dark mode toggle in user settings. + +### Fixed +- Issue where the login button was unresponsive on iOS Safari. +- Memory leak in the analytics background worker. + +### Security +- Updated `axios` to 1.6.0 to address a critical vulnerability. + +## [1.1.0] - 2023-09-15 +### Changed +- Refactored the authentication flow to use JWT instead of sessions. +``` + +### 3. Writing for the Audience +- **Internal/Developer Audience**: Be technical. Include PR numbers, internal IDs, and technical details (e.g., "Optimized SQL query on the `users` table"). +- **External/User Audience**: Be benefit-oriented. Avoid jargon. (e.g., "The app now loads 30% faster on slow connections"). + +### 4. Automation with `semantic-release` +Instead of manual updates, automate the changelog generation based on **Conventional Commits**. + +```bash +# Commit message format: +# feat: add dark mode +# fix: resolve login button issue +``` + +Tools like `semantic-release` or `standard-version` will: +1. Detect the version bump (Major/Minor/Patch). +2. Generate the `CHANGELOG.md` entries automatically. +3. Tag the release in Git. + +## Best Practices +- **Don't just dump Git logs**: A commit history is not a changelog. Commits like "fix typo" or "WIP" should not appear in release notes. +- **Group by Version**: Always list the most recent version at the top. +- **Include Dates**: Every version must have a release date (YYYY-MM-DD). +- **Link to Versions**: If using GitHub, link the version number to the specific tag or diff (e.g., `[1.2.0]: https://github.com/user/repo/compare/v1.1.0...v1.2.0`). +- **Highlight Breaking Changes**: Make them extremely obvious using bold text or a dedicated section. \ No newline at end of file diff --git a/TRAE-Skills/documentation/Code_Comments_Best_Practices.md b/TRAE-Skills/documentation/Code_Comments_Best_Practices.md new file mode 100644 index 0000000000000000000000000000000000000000..b4590f64e01ffd64c8734a8617a185999cc310c9 --- /dev/null +++ b/TRAE-Skills/documentation/Code_Comments_Best_Practices.md @@ -0,0 +1,81 @@ +# Skill: Code Commenting Best Practices + +## Purpose +To write meaningful, concise, and helpful comments that explain the "why" behind complex code, without cluttering the codebase with obvious or redundant information. Good comments act as documentation for future developers (including your future self). + +## When to Use +- When implementing a non-obvious algorithm or workaround +- To document complex business rules that aren't clear from the code alone +- When dealing with technical debt or temporary hacks (e.g., `TODO`, `FIXME`) +- To explain why a specific library or external API is used in a certain way + +## Procedure + +### 1. Types of Comments +- **Documentation Comments (Docstrings)**: Explain *what* a function or class does, its parameters, and return values. +- **Implementation Comments**: Explain *how* or *why* a specific block of code works. +- **TODO/FIXME Comments**: Track pending work or known issues. + +### 2. The "Why", Not the "What" +Code explains the "What" and "How". Comments should explain the "Why". + +**❌ BAD (Redundant)**: +```javascript +// Increment i by 1 +i++; +``` + +**✅ GOOD (Explains intent)**: +```javascript +// We skip the first element because it's a header in the CSV +for (let i = 1; i < data.length; i++) { + // ... +} +``` + +### 3. Using JSDoc (for JavaScript/TypeScript) +Standardize function documentation so editors can provide better IntelliSense. + +```typescript +/** + * Calculates the final price after tax and discounts. + * + * @param price - The base price of the item. + * @param taxRate - The tax rate as a decimal (e.g., 0.22 for 22%). + * @param discount - A fixed discount amount. + * @returns The final price rounded to 2 decimal places. + * + * @example + * calculatePrice(100, 0.22, 10) // returns 112 + */ +function calculatePrice(price: number, taxRate: number, discount: number): number { + // ... +} +``` + +### 4. Handling Workarounds +Always document when you are doing something unusual to fix a bug or handle an edge case. + +```javascript +// NOTE: We use a 50ms timeout here because the third-party modal +// needs time to mount before we can focus the input field. +// See issue #124 for more details. +setTimeout(() => { + inputRef.current.focus(); +}, 50); +``` + +### 5. Managing TODOs +Don't just leave `// TODO: fix this`. Be specific and include your name or a ticket number. + +```javascript +// TODO(JohnDoe): Refactor this to use the new GraphQL API once it's deployed. +// See ticket JIRA-456. +``` + +## Best Practices +- **Good code is self-documenting**: Before writing a comment, ask: "Can I make this code clearer by renaming a variable or extracting a function?" +- **Keep comments up to date**: An incorrect comment is worse than no comment at all. Update comments during refactoring. +- **Avoid "Commented-out code"**: Don't leave old code in comments. That's what Git is for. Delete it. +- **Don't use comments to apologize**: If code is bad, refactor it instead of writing a comment explaining how bad it is. +- **Be professional**: Avoid jokes, frustration, or personal notes in comments. They are part of the permanent codebase. \ No newline at end of file diff --git a/TRAE-Skills/documentation/Component_Documentation_Storybook.md b/TRAE-Skills/documentation/Component_Documentation_Storybook.md new file mode 100644 index 0000000000000000000000000000000000000000..38039cff6be8d48c8c3bfa9d848a37db83d24414 --- /dev/null +++ b/TRAE-Skills/documentation/Component_Documentation_Storybook.md @@ -0,0 +1,108 @@ +# Skill: Component Documentation (Storybook) + +## Purpose +To create an interactive, isolated environment for developing, documenting, and testing UI components. This acts as a living style guide for developers and designers, ensuring consistency across a large application. + +## When to Use +- When building a reusable UI library (e.g., buttons, inputs, cards) +- When collaborating between frontend developers and UI/UX designers +- When documenting edge cases and states (e.g., loading, error, disabled) +- To avoid "copy-pasting" components without knowing their full functionality + +## Procedure + +### 1. Installation +Install Storybook into an existing project (React, Vue, Angular, Svelte, etc.). + +```bash +npx storybook@latest init +``` + +### 2. Creating a Story +Stories represent the different states of a single component. + +**`Button.stories.tsx`** +```tsx +import type { Meta, StoryObj } from '@storybook/react'; +import { Button } from './Button'; + +// 1. Meta Configuration +const meta: Meta = { + title: 'Components/Button', // Categorization in sidebar + component: Button, + tags: ['autodocs'], // Automatically generate documentation page + argTypes: { + backgroundColor: { control: 'color' }, // Add UI controls + onClick: { action: 'clicked' }, // Log actions + }, +}; + +export default meta; +type Story = StoryObj; + +// 2. Define Primary State +export const Primary: Story = { + args: { + label: 'Primary Button', + variant: 'primary', + }, +}; + +// 3. Define Secondary State +export const Secondary: Story = { + args: { + label: 'Secondary Button', + variant: 'secondary', + }, +}; + +// 4. Define Large Button +export const Large: Story = { + args: { + size: 'large', + label: 'Large Button', + }, +}; +``` + +### 3. Adding Documentation (Markdown/MDX) +For more complex components, use MDX to combine Markdown and JSX for rich documentation. + +**`Button.mdx`** +```markdown +import { Meta, Story, Canvas, Controls } from '@storybook/blocks'; +import * as ButtonStories from './Button.stories'; + + + +# Button Component + +The Button is used to trigger an action or event. + +## When to use +- Form submissions +- Navigation (if it looks like a button) +- Triggering modals + + + +## Properties + + +## Examples + +### Secondary + +``` + +### 4. Running Storybook +Launch the development server to view your documentation. +```bash +npm run storybook +``` + +## Best Practices +- **Component Isolation**: Components in Storybook should not depend on global application state (Redux, Context) if possible. Use decorators to provide mock state when necessary. +- **Autodocs**: Enable the `autodocs` tag to get a clean table of props and examples for every component with minimal effort. +- **Visual Testing**: Integrate Storybook with tools like Chromatic for automated visual regression testing of your components. +- **Organize by Category**: Use a clear hierarchy in `title` (e.g., `Atom/Button`, `Molecule/SearchForm`, `Organism/Navbar`). \ No newline at end of file diff --git a/TRAE-Skills/documentation/Contributing_Guidelines_CONTRIBUTING.md b/TRAE-Skills/documentation/Contributing_Guidelines_CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..0d8c8ae15f7189175b2cbef3bf9659d4a141b0c4 --- /dev/null +++ b/TRAE-Skills/documentation/Contributing_Guidelines_CONTRIBUTING.md @@ -0,0 +1,61 @@ +# Skill: Contributing Guidelines (CONTRIBUTING.md) + +## Purpose +To establish clear, welcoming, and standardized procedures for external and internal contributors to collaborate on a project. A well-crafted `CONTRIBUTING.md` minimizes friction, sets expectations, and ensures code quality. + +## When to Use +- When initializing a new open-source or inner-source repository +- When the project starts accepting pull requests from other teams or the public +- When there is confusion about the workflow, commit standards, or testing requirements + +## Procedure + +### 1. Code of Conduct Integration +Always begin by linking to or establishing a Code of Conduct to ensure a safe and inclusive environment. + +### 2. Getting Started & Setup +Provide exact commands to get the project running locally. + +```bash +# Example setup instructions +git clone https://github.com/organization/project.git +cd project +npm install # or pip install -r requirements.txt +cp .env.example .env +npm run dev +``` + +### 3. Branching Strategy +Define the naming conventions for branches. +- `feature/short-description` +- `fix/issue-number-description` +- `docs/what-changed` + +### 4. Commit Message Convention +Enforce a standard, such as Conventional Commits. +``` +type(scope): subject + +body + +footer (e.g., Closes #123) +``` +Valid types: `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `chore`. + +### 5. Pull Request Process +Outline the steps to submit a PR: +1. Ensure all tests pass (`npm run test`). +2. Run linters and formatters (`npm run lint`). +3. Update relevant documentation. +4. Fill out the Pull Request template completely. +5. Request review from code owners. + +### 6. Issue Reporting +Provide templates or guidelines for: +- **Bug Reports**: Expected behavior, actual behavior, steps to reproduce, environment details. +- **Feature Requests**: Problem description, proposed solution, alternatives considered. + +## Best Practices +- Keep it concise but comprehensive. +- Use automated checks (e.g., GitHub Actions, Husky) to enforce these guidelines automatically. +- Provide links to architectural documentation if the project is complex. \ No newline at end of file diff --git a/TRAE-Skills/documentation/Diagramming_Mermaid_JS.md b/TRAE-Skills/documentation/Diagramming_Mermaid_JS.md new file mode 100644 index 0000000000000000000000000000000000000000..fb15127cbeb1382240fef0c41e31147f4c0e00ac --- /dev/null +++ b/TRAE-Skills/documentation/Diagramming_Mermaid_JS.md @@ -0,0 +1,78 @@ +# Skill: Diagramming with Mermaid.js + +## Purpose +To create complex diagrams (flowcharts, sequence diagrams, gantt charts, etc.) directly within Markdown files using a simple, text-based syntax. This ensures documentation and diagrams stay in sync with the code and are easily versionable via Git. + +## When to Use +- When documenting system architecture, data flows, or complex logic +- When explaining authentication sequences (OAuth2, JWT) +- When defining project timelines or state machines +- To avoid using external binary image files for diagrams that change frequently + +## Procedure + +### 1. Basic Flowchart Syntax +Flowcharts represent processes or workflows. + +```mermaid +graph TD + A[Start] --> B{Is logged in?} + B -- Yes --> C[Dashboard] + B -- No --> D[Login Page] + D --> B +``` + +### 2. Sequence Diagrams +Perfect for showing interactions between different services or components over time. + +```mermaid +sequenceDiagram + participant User + participant Frontend + participant AuthServer + participant Database + + User->>Frontend: Click Login + Frontend->>AuthServer: POST /auth/login + AuthServer->>Database: Query User + Database-->>AuthServer: User Data + AuthServer-->>Frontend: JWT Token + Frontend-->>User: Show Dashboard +``` + +### 3. Entity Relationship (ER) Diagrams +Useful for documenting database schemas. + +```mermaid +erDiagram + USER ||--o{ POST : writes + USER { + string username + string email + string password + } + POST { + string title + string content + datetime created_at + } +``` + +### 4. State Diagrams +Best for documenting complex UI states or business logic transitions. + +```mermaid +stateDiagram-v2 + [*] --> Idle + Idle --> Loading: Search clicked + Loading --> Success: Data fetched + Loading --> Error: Fetch failed + Success --> Idle: Clear search + Error --> Loading: Retry +``` + +## Best Practices +- **Keep it Simple**: Don't try to fit too much information into a single diagram. Break complex systems into multiple smaller diagrams. +- **Direction Matters**: Use `TD` (Top-Down) or `LR` (Left-to-Right) consistently based on what makes the flow easier to read. +- **Use meaningful labels**: Instead of `A --> B`, use `Start --> Process`. +- **Git Versioning**: Since Mermaid is text, you can see exact changes in diagrams during code reviews (diffs). Always prefer Mermaid over embedded screenshots of diagrams. \ No newline at end of file diff --git a/TRAE-Skills/documentation/Effective_User_Documentation.md b/TRAE-Skills/documentation/Effective_User_Documentation.md new file mode 100644 index 0000000000000000000000000000000000000000..bc0dbab84d016bc2be8df892e88cea2d3b4e7805 --- /dev/null +++ b/TRAE-Skills/documentation/Effective_User_Documentation.md @@ -0,0 +1,43 @@ +# Skill: Writing Effective User Documentation + +## Purpose +To create guides, tutorials, and help articles that empower non-technical users to successfully use a product, reduce support tickets, and improve overall user satisfaction. + +## When to Use +- When launching a new product or major consumer-facing feature +- When users frequently ask the same questions to support +- To improve user onboarding and self-service capabilities + +## Procedure + +### 1. Know Your Audience +Before writing, identify who you are writing for. +- **Novice**: Needs step-by-step instructions and basic terminology. +- **Intermediate**: Needs specific task-oriented guides. +- **Expert**: Needs advanced tips, shortcuts, and troubleshooting. + +### 2. The Structure of a Good Help Article +1. **Clear, Action-Oriented Title**: (e.g., "How to Reset Your Password", NOT "Password Recovery"). +2. **The "Why"**: A brief sentence explaining what the user will achieve. +3. **Numbered Steps**: Use numbered lists for sequential actions. +4. **Visual Aids**: Use annotated screenshots, GIFs, or short videos for complex steps. +5. **Notes & Warnings**: Use callouts for important information. +6. **Related Articles**: Link to other relevant guides. + +### 3. Writing Style +- **Use Active Voice**: (e.g., "Click the Save button", NOT "The Save button should be clicked"). +- **Be Direct**: Avoid fluff. Start with the most important information. +- **Use Consistent Terminology**: If you call it a "Dashboard", don't call it a "Home Screen" in the next paragraph. +- **Chunk Information**: Use subheaders to break up long articles into scannable sections. + +### 4. The 3-Step Review Process +1. **Technical Review**: Does the feature actually work this way? +2. **Editorial Review**: Is it free of typos and consistent with the brand voice? +3. **Usability Review**: Can a user actually follow these steps to complete the task? (Ideally, have someone who didn't build the feature test it). + +## Best Practices +- **Searchable Content**: Use keywords that users actually type into search engines (e.g., "forgot password", "reset login"). +- **Keep it Updated**: Outdated documentation is worse than no documentation. Set a regular review schedule. +- **Provide "Paths"**: Create "Getting Started" paths for new users rather than just a library of disconnected articles. +- **Accessibility**: Ensure your documentation site is accessible (proper alt text on images, high contrast, keyboard navigable). +- **Feedback Loop**: Allow users to rate articles ("Was this helpful?") to identify content that needs improvement. \ No newline at end of file diff --git a/TRAE-Skills/documentation/Project_Onboarding_Guide.md b/TRAE-Skills/documentation/Project_Onboarding_Guide.md new file mode 100644 index 0000000000000000000000000000000000000000..dce73acebb957607112ff3bfc154de1eebc79f9b --- /dev/null +++ b/TRAE-Skills/documentation/Project_Onboarding_Guide.md @@ -0,0 +1,12 @@ +# Project Onboarding Guide + +## Overview +Reduce the time to first commit for new team members. + +## Contents +1. **Architecture Overview**: High-level diagram. +2. **Environment Setup**: Detailed steps (IDE, extensions, ENV vars). +3. **Key Commands**: How to start, test, build. +4. **Project Structure**: Explanation of folders. +5. **Common Troubleshooting**: Known issues and fixes. +6. **Key Contacts**: Who owns which part of the system. diff --git a/TRAE-Skills/documentation/Technical_Spec_Writing.md b/TRAE-Skills/documentation/Technical_Spec_Writing.md new file mode 100644 index 0000000000000000000000000000000000000000..efec0ce25e7cdad3d229e9deeb1a6f88c99e3623 --- /dev/null +++ b/TRAE-Skills/documentation/Technical_Spec_Writing.md @@ -0,0 +1,49 @@ +# Skill: Technical Specification Writing (RFC/Design Doc) + +## Purpose +To document the architectural decisions, data models, and implementation plans for a complex feature or system before any code is written. This allows for early feedback, cross-team alignment, and avoids costly architectural mistakes. + +## When to Use +- Before starting a task that takes more than 1 week of development +- When introducing a new technology, database, or infrastructure change +- When building features that impact multiple teams or microservices +- To document the rationale behind "why" a specific approach was chosen over others + +## Procedure + +### 1. Document Structure +A standard Technical Spec should include these core sections: + +1. **Overview/Abstract**: 1-2 paragraphs summarizing the problem and the proposed solution. +2. **Background**: Context on why this change is necessary (links to tickets, user research, or performance logs). +3. **Goals & Non-Goals**: + - **Goals**: What we *must* achieve (e.g., "Reduce latency by 50%"). + - **Non-Goals**: Explicitly state what is *out of scope* to prevent scope creep. +4. **Proposed Architecture**: High-level design, diagrams (Mermaid), and data flow. +5. **Data Model**: Schema changes, new tables, or API contracts. +6. **Detailed Implementation Plan**: Step-by-step breakdown of the work. +7. **Alternatives Considered**: Why we didn't choose other approaches (e.g., "We considered MongoDB but chose PostgreSQL because..."). +8. **Security & Performance**: Potential risks and mitigation strategies. +9. **Monitoring & Metrics**: How will we measure success? (e.g., "New Datadog dashboard for error rates"). + +### 2. The "Alternatives Considered" Section +This is the most important section for senior engineers. It demonstrates depth of thought and prevents repeating past mistakes. +- List at least 2 other ways the problem could have been solved. +- Provide a brief pros/cons list for each. +- Explain the specific trade-offs (e.g., "Option A was faster to build but had higher long-term maintenance costs"). + +### 3. Review Process +1. **Draft**: Write the spec in Markdown or a collaborative tool (Google Docs, Notion). +2. **Internal Feedback**: Share with your immediate team for technical sanity checks. +3. **Broad Review**: Share with stakeholders (Product, Security, SRE) for sign-off. +4. **Implementation**: Only start coding once the spec is approved. + +### 4. Updating the Spec +Specs are living documents. If the implementation plan changes significantly during development, update the spec to reflect the reality. + +## Best Practices +- **Write for the Future**: Imagine someone reading this spec 2 years from now. Will they understand *why* these decisions were made? +- **Be Concise**: Use bullet points and diagrams. Avoid "walls of text". +- **Use Clear Language**: Avoid vague terms like "scalable" or "performant". Use numbers (e.g., "Handle 10,000 requests per second"). +- **Link Everything**: Link to PRs, Jira tickets, and related architectural docs (ADRs). +- **Acknowledge Trade-offs**: Every architectural decision has a downside. Be honest about them. \ No newline at end of file diff --git a/TRAE-Skills/documentation/User_Manual_Creation.md b/TRAE-Skills/documentation/User_Manual_Creation.md new file mode 100644 index 0000000000000000000000000000000000000000..097562487dd79b34a28ad949d82819114d1385a1 --- /dev/null +++ b/TRAE-Skills/documentation/User_Manual_Creation.md @@ -0,0 +1,15 @@ +# User Manual Creation + +## Overview +For end-user facing applications, a manual is often necessary. + +## Formats +- **Markdown/GitBook**: Good for technical users. +- **Knowledge Base**: Intercom, Zendesk, Notion. +- **In-App Guides**: Walkthroughs (e.g., Intro.js). + +## Content +- **Getting Started**: Account creation, basic navigation. +- **Features**: Detailed explanation of core features. +- **FAQ**: Answers to common questions. +- **Screenshots/Videos**: Visual aids are critical. diff --git a/TRAE-Skills/documentation/User_Story_Mapping.md b/TRAE-Skills/documentation/User_Story_Mapping.md new file mode 100644 index 0000000000000000000000000000000000000000..2a1a8494bc883c2633d48498e00f62a30e5b4745 --- /dev/null +++ b/TRAE-Skills/documentation/User_Story_Mapping.md @@ -0,0 +1,61 @@ +# Skill: User Story Mapping (Agile) + +## Purpose +To visualize the entire user journey within an application and identify the core features, dependencies, and Minimum Viable Product (MVP) based on user needs rather than a flat backlog of tasks. + +## When to Use +- When initiating a new project or major product feature +- When prioritizing a backlog of features with stakeholders +- When identifying gaps in the user experience (UX) +- To align a development team on the "Big Picture" before starting work + +## Procedure + +### 1. The Concept (Hierarchy) +- **User Personas**: Who are we building this for? (e.g., "End User", "Admin") +- **Activities (The Backbone)**: High-level tasks users perform. (e.g., "Manage Orders") +- **Steps (The Ribs)**: Specific actions within an activity. (e.g., "View Order List", "Edit Order Details", "Cancel Order") +- **User Stories (The Flesh)**: Small, deliverable features. (e.g., "As a user, I want to filter orders by date") + +### 2. Mapping the Journey (Example: E-commerce) +1. **Persona**: Customer +2. **Activities**: Browse Products -> Cart -> Checkout -> Post-Purchase +3. **Steps (Browse Products)**: Search -> View Details -> Filter -> Sort +4. **Stories (Search)**: + - Search by name (MVP) + - Search by category (MVP) + - Search by SKU (Release 2) + - Autocomplete (Release 3) + +### 3. Slicing for MVP +Draw a horizontal line across the map. +- **Above the Line**: The absolute minimum stories needed to complete the user journey. This is your **MVP**. +- **Below the Line**: Future releases, optimizations, and "nice-to-have" features. + +### 4. Refining Stories (INVEST Principle) +Ensure every story in the map is: +- **I**ndependent: Can be developed separately. +- **N**egotiable: Not a rigid contract; open to discussion. +- **V**aluable: Provides clear value to the user. +- **E**stimable: The team can estimate the effort. +- **S**mall: Can be completed within a single sprint. +- **T**estable: Has clear acceptance criteria. + +### 5. Documenting Acceptance Criteria (Gherkin) +For every story, define clear rules for completion. + +```gherkin +Feature: Order Filtering + Scenario: Filter orders by date range + Given I am on the "My Orders" page + And I have 5 orders from last month and 2 from this month + When I select the date range for "This Month" + Then I should see only 2 orders in the list + And the "Total Count" should show 2 +``` + +## Best Practices +- **Involve Everyone**: User Story Mapping should be a collaborative exercise involving Developers, Designers, and Product Owners. +- **Focus on the "Why"**: Every story must have a clear "so that..." statement (e.g., "As a user, I want to filter orders **so that I can quickly find my recent purchases**"). +- **Don't Forget Edge Cases**: Map out error states (e.g., "No results found", "Payment failed") as part of the journey. +- **Keep it Physical (or Digital)**: Use tools like Miro, Mural, or physical sticky notes to make the map visual and interactive. Do not just use a spreadsheet. \ No newline at end of file diff --git a/TRAE-Skills/documentation/Writing_Effective_README.md b/TRAE-Skills/documentation/Writing_Effective_README.md new file mode 100644 index 0000000000000000000000000000000000000000..7e578264d7a43ed08d21e9cdbc1482d79b913b7b --- /dev/null +++ b/TRAE-Skills/documentation/Writing_Effective_README.md @@ -0,0 +1,70 @@ +# Skill: README Best Practices + +## Purpose +To create a high-quality `README.md` that serves as the entry point for a project. A great README explains what the project does, how to get it running, and how to contribute, significantly reducing onboarding time for new developers and stakeholders. + +## When to Use +- When creating a new repository or cleaning up an existing one +- When a project's setup process changes +- To improve the discoverability and usability of a tool or library +- As part of the documentation for a professional portfolio project + +## Procedure + +### 1. The Essential Sections +A professional README should follow a logical order: + +1. **Title & Short Description**: What is this? (Include a logo or badges if applicable). +2. **Key Features**: Why should I use this? (Bullet points). +3. **Demo/Screenshots**: Show it in action. +4. **Getting Started**: + - **Prerequisites**: (e.g., Node.js >= 18, Docker). + - **Installation**: Step-by-step commands. +5. **Usage**: Code examples or CLI commands. +6. **Configuration**: Environment variables or config files. +7. **Architecture (Optional)**: Brief overview or diagram. +8. **Contributing**: Link to `CONTRIBUTING.md`. +9. **License**: Name of the license. + +### 2. Format Example + +```markdown +# 🚀 SuperApp + +A modern task manager built with React and Node.js. + +## ✨ Features +- **Real-time Sync**: Tasks stay in sync across devices. +- **Offline Support**: Work without an internet connection. +- **AI Priority**: Automatically categorizes tasks based on urgency. + +## 🛠️ Getting Started + +### Prerequisites +- Node.js v18.0.0+ +- PostgreSQL v14+ + +### Installation +```bash +git clone https://github.com/user/superapp.git +npm install +cp .env.example .env +npm run dev +``` + +## 📖 Usage +Check out our [Documentation Site](https://docs.superapp.com) for advanced usage guides. +``` + +### 3. Writing Tips +- **Be Concise**: Developers scan READMEs. Use bold text, lists, and code blocks. +- **Use Badges**: Use [Shields.io](https://shields.io/) to show build status, test coverage, and version. +- **Relative Links**: Use relative links for files within the repo (e.g., `[Changelog](./CHANGELOG.md)`) so they work in GitHub's web interface. +- **Table of Contents**: If the README is long, add a TOC at the top. + +## Best Practices +- **Assume Nothing**: Don't assume the user has your specific global tools installed. Include commands like `npx` or `brew install`. +- **Keep it Updated**: A README with broken installation commands is a major red flag. +- **Visuals Matter**: A single GIF of the app in action is worth 10 paragraphs of description. +- **Professional Tone**: Use clear, inclusive language. Avoid "just" or "simply" (e.g., instead of "Simply run this", use "Run this"). +- **Link to Support**: Tell users where to go if they find a bug (GitHub Issues, Slack, etc.). \ No newline at end of file diff --git a/TRAE-Skills/frontend/API_Data_Fetching_TanStack.md b/TRAE-Skills/frontend/API_Data_Fetching_TanStack.md new file mode 100644 index 0000000000000000000000000000000000000000..e6515d77256d2b0a4676d3fdcd2cb66bcdae2f79 --- /dev/null +++ b/TRAE-Skills/frontend/API_Data_Fetching_TanStack.md @@ -0,0 +1,26 @@ +# Skill: API Data Fetching (TanStack Query) + +## Purpose +To handle server state (fetching, caching, synchronizing, and updating) in React applications, replacing manual `useEffect` fetching logic. + +## When to Use +- Fetching data from REST or GraphQL APIs. +- When you need caching, deduplication, background updates, or optimistic UI. + +## Procedure +1. **Setup Provider**: Wrap app in `QueryClientProvider`. +2. **Create Query**: Use `useQuery`. + - Key: Unique array `['todos', { status }]`. + - Fetcher: Async function returning data. + - `const { data, isLoading, error } = useQuery({ queryKey: [...], queryFn: fetchTodos })`. +3. **Create Mutation**: Use `useMutation` for POST/PUT/DELETE. + - `const mutation = useMutation({ mutationFn: addTodo })`. +4. **Invalidate**: On mutation success, invalidate query to refetch fresh data: `queryClient.invalidateQueries({ queryKey: ['todos'] })`. + +## Constraints +- Define query keys consistently. +- Handle loading and error states explicitly in the UI. +- Do not use for local synchronous state (use `useState` or Redux). + +## Expected Output +A data-fetching layer that automatically handles loading states, caching, and re-validation, reducing boilerplate significantly. diff --git a/TRAE-Skills/frontend/Accessibility_Audit.md b/TRAE-Skills/frontend/Accessibility_Audit.md new file mode 100644 index 0000000000000000000000000000000000000000..08a5faf426d64caa26eef1bd8c7833573b2a1475 --- /dev/null +++ b/TRAE-Skills/frontend/Accessibility_Audit.md @@ -0,0 +1,25 @@ +# Skill: Accessibility (a11y) Audit + +## Purpose +To ensure the application is usable by people with disabilities, complying with WCAG standards and improving SEO/UX. + +## When to Use +- During development of UI components. +- Before major releases. + +## Procedure +1. **Semantic HTML**: Use correct tags (`; +} +``` + +### 4. Data Fetching +- **Static (Default)**: `fetch('...', { cache: 'force-cache' })`. +- **Dynamic**: `fetch('...', { cache: 'no-store' })`. +- **Revalidation**: `fetch('...', { next: { revalidate: 3600 } })`. + +## Constraints +- **Context**: You cannot use React Context directly in Server Components. Wrap the provider in a Client Component. +- **Node APIs**: Server components have access to Node APIs (fs, db), but Client components do not. + +## Expected Output +A high-performance application with reduced JavaScript sent to the client and simplified data fetching logic. diff --git a/TRAE-Skills/frontend/PWA_Implementation.md b/TRAE-Skills/frontend/PWA_Implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..eaae89f703946b0cb899f236e3c8b96681cf2bc7 --- /dev/null +++ b/TRAE-Skills/frontend/PWA_Implementation.md @@ -0,0 +1,60 @@ +# Skill: PWA Implementation + +## Purpose +To convert a web application into a Progressive Web App (PWA) that is installable, works offline, and behaves like a native app. + +## When to Use +- When you want a mobile presence without developing separate native apps. +- To improve engagement via "Add to Home Screen". + +## Procedure + +### 1. Manifest File (`manifest.json`) +Describes the app (name, icons, colors). +```json +{ + "name": "My PWA", + "short_name": "PWA", + "start_url": "/", + "display": "standalone", + "background_color": "#ffffff", + "theme_color": "#000000", + "icons": [ + { "src": "/icon-192.png", "sizes": "192x192", "type": "image/png" }, + { "src": "/icon-512.png", "sizes": "512x512", "type": "image/png" } + ] +} +``` +Link it in HTML: ``. + +### 2. Service Worker (Offline Support) +Intercepts network requests. +1. **Register**: + ```javascript + if ('serviceWorker' in navigator) { + navigator.serviceWorker.register('/sw.js'); + } + ``` +2. **Implement (`sw.js`)**: Use Workbox for easier setup, or raw JS. + ```javascript + self.addEventListener('install', (e) => { + e.waitUntil(caches.open('v1').then((cache) => cache.addAll(['/', '/index.html']))); + }); + + self.addEventListener('fetch', (e) => { + e.respondWith(caches.match(e.request).then((res) => res || fetch(e.request))); + }); + ``` + +### 3. HTTPS +PWAs **require** HTTPS (except on localhost). + +### 4. Maskable Icons +Ensure icons have ` "purpose": "any maskable" ` to look good on Android (adaptive icons). + +## Constraints +- **iOS Support**: Limited compared to Android (no push notifications until recently, manual install process). +- **Storage**: Browser cache limits apply (usually ~50-100MB guaranteed, more available if requested). + +## Expected Output +An installable web app with an "App-like" experience and offline capabilities. diff --git a/TRAE-Skills/frontend/React_Component_Optimization.md b/TRAE-Skills/frontend/React_Component_Optimization.md new file mode 100644 index 0000000000000000000000000000000000000000..ea355f39da45ba38c3b56a5ee2f75ebb23408458 --- /dev/null +++ b/TRAE-Skills/frontend/React_Component_Optimization.md @@ -0,0 +1,25 @@ +# Skill: React Component Optimization + +## Purpose +To improve the rendering performance of React applications by preventing unnecessary re-renders of components. + +## When to Use +- When a component renders often (e.g., list items, inputs). +- When a component receives complex props (objects/arrays). +- When profiling reveals "wasted renders". + +## Procedure +1. **Identify Bottlenecks**: Use React DevTools Profiler to find components rendering too frequently. +2. **Memoize Component**: Wrap the component export with `React.memo()`. + - `export const MyComponent = React.memo(function MyComponent(props) { ... });` +3. **Memoize Props**: + - Use `useCallback` for functions passed as props: `const handleClick = useCallback(() => { ... }, [deps])`. + - Use `useMemo` for expensive calculations or object/array props: `const data = useMemo(() => ({ id }), [id])`. +4. **Virtualize Lists**: If the list is huge, use `react-window` or `react-virtualized` to render only visible items. + +## Constraints +- Do not memoize everything blindly; it adds overhead. Only optimize where needed. +- Ensure dependency arrays in hooks are correct (use lint rules). + +## Expected Output +A component structure that only re-renders when its specific props or state actually change, resulting in a smoother UI. diff --git a/TRAE-Skills/frontend/React_Context_vs_Zustand.md b/TRAE-Skills/frontend/React_Context_vs_Zustand.md new file mode 100644 index 0000000000000000000000000000000000000000..916dbee913ffd92abf0ba8cbc7518c09ac63a046 --- /dev/null +++ b/TRAE-Skills/frontend/React_Context_vs_Zustand.md @@ -0,0 +1,59 @@ +# Skill: React Context vs Zustand + +## Purpose +To choose and implement the correct state management strategy for React applications to avoid prop-drilling and complexity. + +## When to Use +- **Context**: Global constants (Theme, User, Locale) that rarely change. +- **Zustand**: Complex application state (Cart, Dashboard Filters) with frequent updates. + +## Procedure + +### 1. React Context (Built-in) +Best for dependency injection. +```tsx +const ThemeContext = createContext('light'); + +export function ThemeProvider({ children }) { + const [theme, setTheme] = useState('light'); + return ( + + {children} + + ); +} +// Usage: const { theme } = useContext(ThemeContext); +``` +**Issue**: When `theme` changes, *all* consumers re-render, even if they only use part of the context. + +### 2. Zustand (External Library) +Simple, fast, scalable. Solves the re-render issue via selectors. +```bash +npm install zustand +``` + +**Store Definition**: +```tsx +import { create } from 'zustand'; + +const useStore = create((set) => ({ + bears: 0, + increasePopulation: () => set((state) => ({ bears: state.bears + 1 })), + removeAllBears: () => set({ bears: 0 }), +})); +``` + +**Usage**: +```tsx +function BearCounter() { + const bears = useStore((state) => state.bears); // Selects only 'bears' + return

{bears} around here ...

; +} +``` + +## Constraints +- **Boilerplate**: Context requires Providers wrapping the app; Zustand does not. +- **DevTools**: Zustand has Redux DevTools support middleware. + +## Expected Output +Optimized rendering performance and cleaner code structure for global state management. diff --git a/TRAE-Skills/frontend/React_Testing_Library.md b/TRAE-Skills/frontend/React_Testing_Library.md new file mode 100644 index 0000000000000000000000000000000000000000..6553d6f64e635384922afa8a0951e6dfb305d60e --- /dev/null +++ b/TRAE-Skills/frontend/React_Testing_Library.md @@ -0,0 +1,53 @@ +# Skill: React Testing Library (RTL) + +## Purpose +To test React components by interacting with them as a user would (clicking, typing, reading text), rather than testing implementation details. + +## When to Use +- Unit and Integration testing of UI components. +- Ensuring accessibility (finding elements by role). + +## Procedure + +### 1. Philosophy +"The more your tests resemble the way your software is used, the more confidence they can give you." +- **Do**: `screen.getByRole('button', { name: /submit/i })` +- **Don't**: `container.querySelector('.btn-primary')` + +### 2. Basic Test +```tsx +import { render, screen, fireEvent } from '@testing-library/react'; +import Counter from './Counter'; + +test('increments counter', () => { + render(); + + // Assert initial state + expect(screen.getByText('Count: 0')).toBeInTheDocument(); + + // Act + fireEvent.click(screen.getByRole('button', { name: /increment/i })); + + // Assert new state + expect(screen.getByText('Count: 1')).toBeInTheDocument(); +}); +``` + +### 3. User Event (Recommended) +`user-event` simulates real browser interactions better than `fireEvent`. +```tsx +import userEvent from '@testing-library/user-event'; + +test('typing', async () => { + const user = userEvent.setup(); + render(); + await user.type(screen.getByRole('textbox'), 'Hello'); +}); +``` + +## Constraints +- **Async**: Use `findBy...` (returns Promise) for elements that appear after an API call/timeout. +- **Mocking**: Mock external modules (API calls) using Jest/Vitest `vi.mock` to keep tests fast and isolated. + +## Expected Output +Robust test suite ensuring components behave correctly from the user's perspective. diff --git a/TRAE-Skills/frontend/Responsive_UI_Design_Tailwind.md b/TRAE-Skills/frontend/Responsive_UI_Design_Tailwind.md new file mode 100644 index 0000000000000000000000000000000000000000..81a0b05513fedd67ab870d6a7dce51d1c473e38a --- /dev/null +++ b/TRAE-Skills/frontend/Responsive_UI_Design_Tailwind.md @@ -0,0 +1,25 @@ +# Skill: Responsive UI Design (Tailwind CSS) + +## Purpose +To create user interfaces that adapt seamlessly to different screen sizes (Mobile, Tablet, Desktop) using a utility-first CSS framework. + +## When to Use +- Building any user-facing layout. +- Ensuring mobile compatibility. + +## Procedure +1. **Mobile First**: Design for the smallest screen first (default classes). + - `
` (Mobile default). +2. **Add Breakpoints**: Use prefixes (`sm:`, `md:`, `lg:`, `xl:`) to override styles on larger screens. + - `
` (Block on mobile, Flex on tablet+). +3. **Grid/Flex Layouts**: + - Use `grid-cols-1 md:grid-cols-2 lg:grid-cols-3` for responsive grids. +4. **Spacing/Typography**: Adjust padding and font size for readability on different devices. + - `p-4 md:p-8`, `text-sm md:text-base`. + +## Constraints +- Do not use fixed widths (`w-96`) unless necessary; use percentages or `max-w` containers. +- Test on actual devices or DevTools device mode, not just resizing the browser window. + +## Expected Output +A UI component that looks and functions correctly across a range of device viewports. diff --git a/TRAE-Skills/frontend/Route_Protection_React_Router.md b/TRAE-Skills/frontend/Route_Protection_React_Router.md new file mode 100644 index 0000000000000000000000000000000000000000..928f9e8a0c7479e17c648fa486865c087a821aff --- /dev/null +++ b/TRAE-Skills/frontend/Route_Protection_React_Router.md @@ -0,0 +1,26 @@ +# Skill: Route Protection (React Router) + +## Purpose +To restrict access to specific pages (routes) based on user authentication status or roles, redirecting unauthorized users. + +## When to Use +- Dashboards, settings pages, or any private content. +- Redirecting logged-in users away from Login/Signup pages. + +## Procedure +1. **Create Wrapper**: Create a `ProtectedRoute` component. + - Props: `children` (the page to render). +2. **Check Auth**: Read auth state (from Context/Redux). +3. **Conditional Render**: + - If `isAuthenticated`, return `children`. + - If not, return ``. +4. **Usage**: Wrap routes in the router definition. + - `} />`. +5. **Role Check** (Optional): Extend to check `user.role` for admin routes. + +## Constraints +- Always handle the "loading" state of authentication (don't redirect while checking if the user is logged in). +- Remember: Client-side protection is UX only; the API must still be secured. + +## Expected Output +A routing configuration that automatically guards private pages and handles redirects for unauthenticated users. diff --git a/TRAE-Skills/frontend/SVG_Animation_Techniques.md b/TRAE-Skills/frontend/SVG_Animation_Techniques.md new file mode 100644 index 0000000000000000000000000000000000000000..690ba9669cc4a7411dd3c4d442cb5fdc5a48b5fe --- /dev/null +++ b/TRAE-Skills/frontend/SVG_Animation_Techniques.md @@ -0,0 +1,62 @@ +# Skill: SVG Animation Techniques + +## Purpose +To create scalable, crisp animations for icons, logos, and illustrations using CSS or JavaScript libraries. + +## When to Use +- Loading spinners. +- Interactive icons (hamburger menu to X). +- Complex illustrations (paths drawing themselves). + +## Procedure + +### 1. CSS Animation (Simple) +Manipulate properties like `opacity`, `transform`, `stroke-dasharray`. +```css +/* Draw a line effect */ +.path { + stroke-dasharray: 100; + stroke-dashoffset: 100; + animation: draw 2s forwards; +} + +@keyframes draw { + to { stroke-dashoffset: 0; } +} +``` + +### 2. SMIL (Native SVG) +Using `` tags inside SVG. Deprecated in some contexts but powerful. +```xml + + + +``` + +### 3. GSAP (GreenSock) - Standard for Complex Motion +Robust, cross-browser, high performance. +```javascript +import gsap from 'gsap'; + +gsap.to(".box", { + rotation: 360, + x: 100, + duration: 1 +}); +``` + +### 4. Framer Motion (React) +Declarative animations. +```tsx + +``` + +## Constraints +- **Complexity**: Animating many DOM nodes/SVG elements can cause layout thrashing. +- **Optimization**: Optimize SVGs (SVGO) before animating to remove useless metadata and group elements properly. + +## Expected Output +Smooth, resolution-independent animations that enhance UI interactivity. diff --git a/TRAE-Skills/frontend/Storybook_Component_Documentation.md b/TRAE-Skills/frontend/Storybook_Component_Documentation.md new file mode 100644 index 0000000000000000000000000000000000000000..5b209b75085b14da8197bf346c1c8dec5ee7bed8 --- /dev/null +++ b/TRAE-Skills/frontend/Storybook_Component_Documentation.md @@ -0,0 +1,60 @@ +# Skill: Storybook Component Documentation + +## Purpose +To develop, test, and document UI components in isolation from the main application. + +## When to Use +- Building a Design System. +- Visual testing. +- Collaborating with designers. + +## Procedure + +### 1. Setup +```bash +npx storybook@latest init +``` + +### 2. Writing Stories +A story captures a specific state of a component. +`Button.stories.tsx`: +```tsx +import type { Meta, StoryObj } from '@storybook/react'; +import { Button } from './Button'; + +const meta: Meta = { + component: Button, + title: 'Components/Button', +}; +export default meta; + +type Story = StoryObj; + +export const Primary: Story = { + args: { + primary: true, + label: 'Button', + }, +}; + +export const Secondary: Story = { + args: { + label: 'Button', + }, +}; +``` + +### 3. Addons +- **Controls**: Edit props dynamically in the UI. +- **Actions**: Log events (clicks). +- **Docs**: Auto-generate documentation from comments (JSDoc). + +### 4. Testing +Use Interaction Testing (play function) to simulate user behavior within Storybook. + +## Constraints +- **Maintenance**: Stories must be updated when component props change. +- **Context**: Components relying on global providers (Redux, Router) need decorators in `.storybook/preview.js` to work. + +## Expected Output +A living style guide where every component state is cataloged and interactive. diff --git a/TRAE-Skills/frontend/WebGL_Advanced_Threejs.md b/TRAE-Skills/frontend/WebGL_Advanced_Threejs.md new file mode 100644 index 0000000000000000000000000000000000000000..9f7a8e4c27d355e6add219c59b2882da6be5ee93 --- /dev/null +++ b/TRAE-Skills/frontend/WebGL_Advanced_Threejs.md @@ -0,0 +1,165 @@ +# Skill: Advanced WebGL with Three.js + +## Purpose +To build high-performance 3D graphics, animations, and interactive experiences using Three.js, including shaders, post-processing, and advanced rendering techniques. + +## When to Use +- When building 3D product configurators or visualizers +- For creating interactive data visualizations in 3D +- When developing games or game-like experiences for the web +- For building virtual tours or architectural visualizations +- When implementing advanced particle effects or simulations + +## Procedure + +### 1. Custom Shaders (GLSL) +Create custom vertex and fragment shaders. + +```javascript +import * as THREE from 'three'; + +const vertexShader = ` + varying vec2 vUv; + varying vec3 vPosition; + + void main() { + vUv = uv; + vPosition = position; + gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); + } +`; + +const fragmentShader = ` + varying vec2 vUv; + varying vec3 vPosition; + uniform float time; + + void main() { + vec3 color = 0.5 + 0.5 * cos(time + vPosition.xyx + vec3(0.0, 2.0, 4.0)); + gl_FragColor = vec4(color, 1.0); + } +`; + +const material = new THREE.ShaderMaterial({ + vertexShader, + fragmentShader, + uniforms: { + time: { value: 0 } + } +}); + +const geometry = new THREE.BoxGeometry(1, 1, 1); +const mesh = new THREE.Mesh(geometry, material); +scene.add(mesh); + +// Animate +function animate() { + requestAnimationFrame(animate); + material.uniforms.time.value += 0.01; + renderer.render(scene, camera); +} +``` + +### 2. Post-Processing Effects +Use post-processing for bloom, depth of field, and other effects. + +```javascript +import { EffectComposer } from 'three/examples/jsm/postprocessing/EffectComposer.js'; +import { RenderPass } from 'three/examples/jsm/postprocessing/RenderPass.js'; +import { UnrealBloomPass } from 'three/examples/jsm/postprocessing/UnrealBloomPass.js'; + +const composer = new EffectComposer(renderer); +composer.addPass(new RenderPass(scene, camera)); + +const bloomPass = new UnrealBloomPass( + new THREE.Vector2(window.innerWidth, window.innerHeight), + 1.5, // strength + 0.4, // radius + 0.85 // threshold +); +composer.addPass(bloomPass); + +// Animate with composer +function animate() { + requestAnimationFrame(animate); + composer.render(); +} +``` + +### 3. Particle Systems +Create advanced particle effects. + +```javascript +const particleCount = 10000; +const positions = new Float32Array(particleCount * 3); +const colors = new Float32Array(particleCount * 3); + +for (let i = 0; i < particleCount; i++) { + positions[i * 3] = (Math.random() - 0.5) * 10; + positions[i * 3 + 1] = (Math.random() - 0.5) * 10; + positions[i * 3 + 2] = (Math.random() - 0.5) * 10; + + colors[i * 3] = Math.random(); + colors[i * 3 + 1] = Math.random(); + colors[i * 3 + 2] = Math.random(); +} + +const geometry = new THREE.BufferGeometry(); +geometry.setAttribute('position', new THREE.BufferAttribute(positions, 3)); +geometry.setAttribute('color', new THREE.BufferAttribute(colors, 3)); + +const material = new THREE.PointsMaterial({ + size: 0.02, + vertexColors: true +}); + +const particles = new THREE.Points(geometry, material); +scene.add(particles); +``` + +### 4. Physics Integration (Cannon.js) +Add physics to your 3D scene. + +```javascript +import * as CANNON from 'cannon-es'; + +const world = new CANNON.World({ + gravity: new CANNON.Vec3(0, -9.82, 0) +}); + +// Create ground +const groundBody = new CANNON.Body({ + shape: new CANNON.Plane(), + mass: 0 +}); +groundBody.quaternion.setFromAxisAngle(new CANNON.Vec3(1, 0, 0), -Math.PI / 2); +world.addBody(groundBody); + +// Create falling sphere +const sphereBody = new CANNON.Body({ + mass: 5, + shape: new CANNON.Sphere(0.5) +}); +sphereBody.position.set(0, 5, 0); +world.addBody(sphereBody); + +// Update physics in animation loop +function animate() { + requestAnimationFrame(animate); + world.step(1 / 60); + + // Sync Three.js mesh with physics body + mesh.position.copy(sphereBody.position); + mesh.quaternion.copy(sphereBody.quaternion); + + renderer.render(scene, camera); +} +``` + +## Best Practices +- **Performance**: Use BufferGeometry instead of Geometry for better performance +- **LOD**: Implement Level of Detail for distant objects +- **Textures**: Compress textures and use appropriate sizes +- **Memory**: Dispose of unused geometries, materials, and textures +- **Shaders**: Optimize shaders by minimizing expensive operations +- **Responsive**: Handle window resizing and device pixel ratio correctly diff --git a/TRAE-Skills/frontend/Web_Animations_Framer_Motion.md b/TRAE-Skills/frontend/Web_Animations_Framer_Motion.md new file mode 100644 index 0000000000000000000000000000000000000000..f8f2f5d57d085fe6ddc069657039816907692d54 --- /dev/null +++ b/TRAE-Skills/frontend/Web_Animations_Framer_Motion.md @@ -0,0 +1,212 @@ +# Skill: Advanced Web Animations with Framer Motion + +## Purpose +To create smooth, production-ready animations and transitions in React applications using Framer Motion. + +## When to Use +- When building modern, polished UIs with animations +- For page transitions and navigation animations +- When implementing interactive micro-interactions +- For scroll-triggered animations +- When creating engaging onboarding experiences + +## Procedure + +### 1. Basic Animations +Simple animations with Framer Motion. + +```jsx +import { motion } from 'framer-motion'; + +function BasicAnimation() { + return ( + + Hello, World! + + ); +} + +function HoverAnimation() { + return ( + + Click Me + + ); +} +``` + +### 2. Variants for Complex Animations +Define reusable animation variants. + +```jsx +import { motion } from 'framer-motion'; + +const containerVariants = { + hidden: { opacity: 0 }, + visible: { + opacity: 1, + transition: { + staggerChildren: 0.1 + } + } +}; + +const itemVariants = { + hidden: { opacity: 0, y: 20 }, + visible: { + opacity: 1, + y: 0, + transition: { duration: 0.5 } + } +}; + +function StaggeredList() { + const items = ['Item 1', 'Item 2', 'Item 3', 'Item 4']; + + return ( + + {items.map((item, index) => ( + + {item} + + ))} + + ); +} +``` + +### 3. Page Transitions +Animate transitions between pages. + +```jsx +import { AnimatePresence, motion } from 'framer-motion'; +import { useLocation, Routes, Route } from 'react-router-dom'; + +const pageVariants = { + initial: { opacity: 0, x: '-100vw' }, + in: { opacity: 1, x: 0 }, + out: { opacity: 0, x: '100vw' } +}; + +const pageTransition = { + type: 'tween', + ease: 'anticipate', + duration: 0.5 +}; + +function App() { + const location = useLocation(); + + return ( + + + + + + } + /> + + + + } + /> + + + ); +} +``` + +### 4. Scroll-Triggered Animations +Animate elements as they scroll into view. + +```jsx +import { motion, useScroll, useTransform } from 'framer-motion'; + +function ParallaxSection() { + const { scrollYProgress } = useScroll(); + const y = useTransform(scrollYProgress, [0, 1], [0, -200]); + const opacity = useTransform(scrollYProgress, [0, 0.5], [1, 0]); + + return ( + +

Scroll Down

+ + ); +} + +function ScrollReveal({ children }) { + return ( + + {children} + + ); +} +``` + +### 5. Gestures and Drag +Implement drag-and-drop interactions. + +```jsx +import { motion } from 'framer-motion'; + +function DraggableCard() { + return ( + + Drag Me! + + ); +} +``` + +## Best Practices +- **Performance**: Use `will-change` for expensive animations +- **Reduced Motion**: Respect user's reduced motion preferences with `useReducedMotion` +- **Variants**: Use variants for reusable animations +- **AnimatePresence**: Always use AnimatePresence for exit animations +- **Spring Physics**: Prefer spring animations for natural feel +- **Test on Mobile**: Ensure animations work well on mobile devices +- **Don't Overdo It**: Use animations purposefully, not just for decoration diff --git a/TRAE-Skills/frontend/Web_Vitals_Optimization.md b/TRAE-Skills/frontend/Web_Vitals_Optimization.md new file mode 100644 index 0000000000000000000000000000000000000000..8645f6605fca745b527fa3fab4600d7b13fede0e --- /dev/null +++ b/TRAE-Skills/frontend/Web_Vitals_Optimization.md @@ -0,0 +1,44 @@ +# Skill: Web Vitals Optimization + +## Purpose +To measure and improve Core Web Vitals (LCP, INP, CLS) for better user experience and SEO rankings. + +## When to Use +- When users complain about slowness. +- Preparing for a production launch. +- Continuous monitoring. + +## Procedure + +### 1. Measurement +- **Lighthouse**: Lab data. Run in Chrome DevTools. +- **Web Vitals Extension**: Real-time checking. +- **Code**: + ```javascript + import { onLCP, onINP, onCLS } from 'web-vitals'; + onLCP(console.log); + ``` + +### 2. Optimizing LCP (Largest Contentful Paint) +Goal: < 2.5s. +- **Images**: Use WebP/AVIF. Explicit `width`/`height`. +- **Preload**: ``. +- **Server**: Improve TTFB (Time to First Byte) via caching/CDN. + +### 3. Optimizing INP (Interaction to Next Paint) +Goal: < 200ms. Replaces FID. +- **Main Thread**: Break up long tasks using `setTimeout` or `requestIdleCallback`. +- **React**: Use `useTransition` for non-urgent updates. +- **Events**: Avoid heavy logic in event handlers. + +### 4. Optimizing CLS (Cumulative Layout Shift) +Goal: < 0.1. +- **Space**: Always reserve space for images/ads/embeds (CSS `aspect-ratio`). +- **Fonts**: Use `font-display: swap` to avoid FOIT (Flash of Invisible Text). + +## Constraints +- **Third-Party Scripts**: Analytics/Ads often hurt performance. Defer them (` + + + diff --git a/frontend/metadata.json b/frontend/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3e62aee5e59688c791e669169cd8d5ac3b6d0a65 --- /dev/null +++ b/frontend/metadata.json @@ -0,0 +1,6 @@ +{ + "name": "", + "description": "", + "requestFramePermissions": [], + "majorCapabilities": ["MAJOR_CAPABILITY_SERVER_SIDE_GEMINI_API"] +} diff --git a/frontend/package-lock.json b/frontend/package-lock.json new file mode 100644 index 0000000000000000000000000000000000000000..39bffa123815003daa79e873624ce5cd4e29c212 --- /dev/null +++ b/frontend/package-lock.json @@ -0,0 +1,4364 @@ +{ + "name": "react-example", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "react-example", + "version": "0.0.0", + "dependencies": { + "@google/genai": "^2.4.0", + "@tailwindcss/vite": "^4.1.14", + "@vitejs/plugin-react": "^5.0.4", + "clsx": "^2.1.1", + "dotenv": "^17.2.3", + "express": "^4.21.2", + "lucide-react": "^0.546.0", + "motion": "^12.23.24", + "react": "^19.0.1", + "react-dom": "^19.0.1", + "react-router-dom": "^7.18.1", + "tailwind-merge": "^3.6.0", + "vite": "^6.2.3" + }, + "devDependencies": { + "@types/express": "^4.17.21", + "@types/node": "^22.14.0", + "autoprefixer": "^10.4.21", + "esbuild": "^0.25.0", + "tailwindcss": "^4.1.14", + "tsx": "^4.21.0", + "typescript": "~5.8.2", + "vite": "^6.2.3" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", + "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", + "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.7" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.29.7.tgz", + "integrity": "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.29.7.tgz", + "integrity": "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", + "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@google/genai": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-2.10.0.tgz", + "integrity": "sha512-e4cFxj3tiuMtsgOT4G9c1hXyGJhg7/Buj7VVeBacRY3fRtkRZZ59Q3nuVp2xbq8BGQXLXCDB253qMhklMOeUDg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^10.3.0", + "p-retry": "^4.6.2", + "protobufjs": "^7.5.4", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.3.tgz", + "integrity": "sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q==", + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.2.tgz", + "integrity": "sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.62.2.tgz", + "integrity": "sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.62.2.tgz", + "integrity": "sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.62.2.tgz", + "integrity": "sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.62.2.tgz", + "integrity": "sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.62.2.tgz", + "integrity": "sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.62.2.tgz", + "integrity": "sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.62.2.tgz", + "integrity": "sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.62.2.tgz", + "integrity": "sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.62.2.tgz", + "integrity": "sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.62.2.tgz", + "integrity": "sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.62.2.tgz", + "integrity": "sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.62.2.tgz", + "integrity": "sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.62.2.tgz", + "integrity": "sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.62.2.tgz", + "integrity": "sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.62.2.tgz", + "integrity": "sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.62.2.tgz", + "integrity": "sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.62.2.tgz", + "integrity": "sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.62.2.tgz", + "integrity": "sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.62.2.tgz", + "integrity": "sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.62.2.tgz", + "integrity": "sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.62.2.tgz", + "integrity": "sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.62.2.tgz", + "integrity": "sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.62.2.tgz", + "integrity": "sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.62.2.tgz", + "integrity": "sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@tailwindcss/node": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.2.tgz", + "integrity": "sha512-yWP/sqEcBLaD8JuA6zNwxoYKr75qxTioYwlRwekj5Jr/I5GXnoJfjetH/psLUIv74cYTH2lBUEzBkinthoYcBg==", + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "enhanced-resolve": "5.21.6", + "jiti": "^2.7.0", + "lightningcss": "1.32.0", + "magic-string": "^0.30.21", + "source-map-js": "^1.2.1", + "tailwindcss": "4.3.2" + } + }, + "node_modules/@tailwindcss/oxide": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.3.2.tgz", + "integrity": "sha512-z8ZgnzX8gdNoWLBLqBPoh/sjnxkwvf9ZuWjnO0l0yIzbLa5/9S+eC5QxGZKRobVHIC3/1BoMWjHblqWjcgFgag==", + "license": "MIT", + "engines": { + "node": ">= 20" + }, + "optionalDependencies": { + "@tailwindcss/oxide-android-arm64": "4.3.2", + "@tailwindcss/oxide-darwin-arm64": "4.3.2", + "@tailwindcss/oxide-darwin-x64": "4.3.2", + "@tailwindcss/oxide-freebsd-x64": "4.3.2", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.3.2", + "@tailwindcss/oxide-linux-arm64-gnu": "4.3.2", + "@tailwindcss/oxide-linux-arm64-musl": "4.3.2", + "@tailwindcss/oxide-linux-x64-gnu": "4.3.2", + "@tailwindcss/oxide-linux-x64-musl": "4.3.2", + "@tailwindcss/oxide-wasm32-wasi": "4.3.2", + "@tailwindcss/oxide-win32-arm64-msvc": "4.3.2", + "@tailwindcss/oxide-win32-x64-msvc": "4.3.2" + } + }, + "node_modules/@tailwindcss/oxide-android-arm64": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.3.2.tgz", + "integrity": "sha512-WHxqIuHpvZ5VtdX6GTl1Ik/Vp2YuN42Et+0CdeaVd/frQ9jAvGmvR8vLT+jk3e8/Q3x8kECB9+R17pgpp2BulA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-arm64": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.3.2.tgz", + "integrity": "sha512-GZypeUY/IDJW3877KeM+O67vbXr3MBnbtEL4aYhNErv/JWZhye2vGSWWG9tB6iiqR2MqRNkY8IOUy4NdSZV26w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-x64": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.3.2.tgz", + "integrity": "sha512-UIIzmefR6KO1sDU7MzRqAxC8iBpft/VhkGjTjnhoS6k7Z3rQ9wEgA1ODSiyH/tcSYssulNm4Ci3hOeK1jH7ccQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-freebsd-x64": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.3.2.tgz", + "integrity": "sha512-GN+uAmcI6DNspnCDwtOAZrTz6oukJnp337qZvxqCGLd3BHBzJpO0ZbTLRvJNdztOeAmTzewewGIMPb0tk2R4WA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.3.2.tgz", + "integrity": "sha512-4ABn7qSbdHRwTiDiuWNegCyb5+2FJ4vKIKc3DmKrvAFw7MU1Lm11dIkTPwUaFdTzc7IsOpDbqBrlh0x6y36U/w==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.3.2.tgz", + "integrity": "sha512-wDgEIGwoM8w8pufh9LVt1PahDgNdKXrLC2qfAnV3vAmococ9RWbxeAw4pxPttd/TsJfwjyLf90Dg1y9y8I6Emw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-musl": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.3.2.tgz", + "integrity": "sha512-J5Nuk0uZQIiMTJj3LEx4sAA9tMFUoXQZFv1J6An+QGYe53HKRJuFDi0rpq/tuouCZeAbOBY3kQ6g8qeD4TUjtA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-gnu": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.3.2.tgz", + "integrity": "sha512-kqCZpSKOBEJO4mz7OqWoofBZeXTAwaVGPj0ErAj7CojmhKpWVWVOnrt9dE8odoIraZq4oj3ausM37kXi+Tow8w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-musl": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.3.2.tgz", + "integrity": "sha512-cixpqbh2toJDmkuCRI68nXA8ZxNmdK9Y+9v5h3MC3ZQKy/0BO8AWzlkWyRM7JAFSGBlfig4YVTPsK6MVgqz1uw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.3.2.tgz", + "integrity": "sha512-4ec2Z/LOmRsAgU23CS4xeJfcJlmRg94A/XrbGRCF1gyU/zdDfRLYDVsS+ynSZCmGNxQ1jQriQOKMQeQxBA3Isw==", + "bundleDependencies": [ + "@napi-rs/wasm-runtime", + "@emnapi/core", + "@emnapi/runtime", + "@tybys/wasm-util", + "@emnapi/wasi-threads", + "tslib" + ], + "cpu": [ + "wasm32" + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.11.1", + "@emnapi/runtime": "^1.11.1", + "@emnapi/wasi-threads": "^1.2.2", + "@napi-rs/wasm-runtime": "^1.1.4", + "@tybys/wasm-util": "^0.10.2", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.2.tgz", + "integrity": "sha512-Zyr/M0+XcYZu3bZrUytc7TXvrk0ftWfl8gN2MwekNDzhqhKRUucMPSeOzM0o0wH5AWOU49BsKRrfKxI2atCPMQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-win32-x64-msvc": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.3.2.tgz", + "integrity": "sha512-QI9BO7KlNZsp2GuO0jwAAj5jCDABOKXRkCk2XuKTSaNEFSdfzqswYVTtCHBNKHLsqyjFyFkqlDiwkNbTYSssMQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/vite": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.3.2.tgz", + "integrity": "sha512-eHpMeX4JXfVNJDEcsouTeCBubJBTcTLigeaw/NTUW6PB5ATKKXdyonnXgTBX2VuRbjz1hjfz6C5XAhr52ImQXA==", + "license": "MIT", + "dependencies": { + "@tailwindcss/node": "4.3.2", + "@tailwindcss/oxide": "4.3.2", + "tailwindcss": "4.3.2" + }, + "peerDependencies": { + "vite": "^5.2.0 || ^6 || ^7 || ^8" + } + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/body-parser": { + "version": "1.19.6", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", + "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "license": "MIT" + }, + "node_modules/@types/express": { + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", + "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "^1" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.19.8", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz", + "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/http-errors": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", + "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.0.tgz", + "integrity": "sha512-QWlFW2wf3nTjC13/DqRnBpR4ZO36VJH/JVBkA/vcnmbTBNQIlnObqyqZE1tUR7+Ni23Lda8R1BxMfbXRpCUx5g==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/qs": { + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", + "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", + "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "<1" + } + }, + "node_modules/@types/serve-static/node_modules/@types/send": { + "version": "0.17.6", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", + "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.2.0.tgz", + "integrity": "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.29.0", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-rc.3", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.18.0" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/autoprefixer": { + "version": "10.5.2", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.5.2.tgz", + "integrity": "sha512-rD5t5DwOjJdmSORcTq64j8MawTC+tbQ+HHqjR4NDumamy/ambn1UJrlKL+KdwujWxMkFjPM3pPHOEA9tl4767Q==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/autoprefixer" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "browserslist": "^4.28.4", + "caniuse-lite": "^1.0.30001799", + "fraction.js": "^5.3.4", + "picocolors": "^1.1.1", + "postcss-value-parser": "^4.2.0" + }, + "bin": { + "autoprefixer": "bin/autoprefixer" + }, + "engines": { + "node": "^10 || ^12 || >=14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.10.42", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.42.tgz", + "integrity": "sha512-c/jurFrDLyui7o1J86yLkRu4LMsTYcBohveus7/I2Hzdn9KIP2bdJPTue/lR1KH46enoPbD77GKeSYNdyPoD3Q==", + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/body-parser": { + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/browserslist": { + "version": "4.28.5", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.5.tgz", + "integrity": "sha512-Cu2E6QejHWzuDMTkuwgpABFgDfZrXLQq5V13YOACZx4mFAG4IwGTbTfHPMr4WtxlHoXSM8FIuRwYYCz5XiabaQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "baseline-browser-mapping": "^2.10.42", + "caniuse-lite": "^1.0.30001800", + "electron-to-chromium": "^1.5.387", + "node-releases": "^2.0.50", + "update-browserslist-db": "^1.2.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001802", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001802.tgz", + "integrity": "sha512-vmv8ub2xwTNmljSKf82mtCk5JH7hC+YgzLj3P5zotvA0tPQ9016tdNNOG8WRca1IxOnhSsivB+J0z5FeE5LOUw==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "license": "MIT" + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.387", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.387.tgz", + "integrity": "sha512-TaxwufTFDufvPEoXdhwVrA3UdFWBeWGkYoJ1K8ldF1xe6gKfth6iRNS5lTQ5JPNOHdGQm8PT1QYKUqFLCiUefQ==", + "license": "ISC" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/enhanced-resolve": { + "version": "5.21.6", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.21.6.tgz", + "integrity": "sha512-aNnGCvbJ/RIyWo1IuhNdVjnNF+EjH9wpzpNHt+ci/m9He9LJvUN8wrCcXjp9cWsGNAuvSpVFTx/vraAFQ8qGjQ==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.3.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.22.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", + "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.5", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.15.1", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fraction.js": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz", + "integrity": "sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/rawify" + } + }, + "node_modules/framer-motion": { + "version": "12.42.2", + "resolved": "https://registry.npmjs.org/framer-motion/-/framer-motion-12.42.2.tgz", + "integrity": "sha512-5XY9luDiu0oHfHBjpDthFMh0ES+122w6p/papSJBweMkO8Sn+PW2QaEgRblQBpWFnuvZS5qvarpt/hO2pjGmnw==", + "license": "MIT", + "dependencies": { + "motion-dom": "^12.42.2", + "motion-utils": "^12.39.0", + "tslib": "^2.4.0" + }, + "peerDependencies": { + "@emotion/is-prop-valid": "*", + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/is-prop-valid": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.6.tgz", + "integrity": "sha512-aIQ0QL8Or8vsUhHyXGA6AohOFRrAAiHhrvsAG6myzcSlfhxSXtnwXA/pRuQTilFgjhLe30swK5rg1d7E1f8Izw==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/google-auth-library": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.9.0.tgz", + "integrity": "sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/lucide-react": { + "version": "0.546.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-0.546.0.tgz", + "integrity": "sha512-Z94u6fKT43lKeYHiVyvyR8fT7pwCzDu7RyMPpTvh054+xahSgj4HFQ+NmflvzdXsoAjYGdCguGaFKYuvq0ThCQ==", + "license": "ISC", + "peerDependencies": { + "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/motion": { + "version": "12.42.2", + "resolved": "https://registry.npmjs.org/motion/-/motion-12.42.2.tgz", + "integrity": "sha512-Atvv11yUKIid41cVrRBDVX5m8tF8kNpExRSlbpt6APClhDjtwQssgFHhQzejxw7/7YYbjHSPKBVbHo05BuJT5Q==", + "license": "MIT", + "dependencies": { + "framer-motion": "^12.42.2", + "tslib": "^2.4.0" + }, + "peerDependencies": { + "@emotion/is-prop-valid": "*", + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/is-prop-valid": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, + "node_modules/motion-dom": { + "version": "12.42.2", + "resolved": "https://registry.npmjs.org/motion-dom/-/motion-dom-12.42.2.tgz", + "integrity": "sha512-5gIMWLp/PycBtJRJWRgjxke5n8dlvkSn2DrYW+tr3XcqAZY1xZh6BJyooJXCM8wdfM7wfMjkBJNLge1CKPUIRA==", + "license": "MIT", + "dependencies": { + "motion-utils": "^12.39.0" + } + }, + "node_modules/motion-utils": { + "version": "12.39.0", + "resolved": "https://registry.npmjs.org/motion-utils/-/motion-utils-12.39.0.tgz", + "integrity": "sha512-8nadJAJjTtqRkmRF36FoJTrywK9nnFmnPwnSMyxaOCU7GDjN9RTMJIxx9De8ErM+vpPhMccr/6fo5WciyQLnMQ==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.15", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", + "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/node-releases": { + "version": "2.0.50", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.50.tgz", + "integrity": "sha512-J6l92tKHX6w8Jy5nO1Vuc01NoIiRGi/d6qBKVxh+IQ8Cr3b6HbVNfKiF8ZpFKufTwpwxMmce2W3iQZ861ZRyTg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.16", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", + "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/react": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.7.tgz", + "integrity": "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz", + "integrity": "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "scheduler": "^0.27.0" + }, + "peerDependencies": { + "react": "^19.2.7" + } + }, + "node_modules/react-refresh": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.18.0.tgz", + "integrity": "sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-router": { + "version": "7.18.1", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.1.tgz", + "integrity": "sha512-GDLgg3i3uM0aeJO3Fm+TCS+sDQ7gu12T6x0qdTEzcwqEfleci7JwugVNIF3U//0FWKnJT7ptG+20B2jfDqnZAg==", + "license": "MIT", + "dependencies": { + "cookie": "^1.0.1", + "set-cookie-parser": "^2.6.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + } + } + }, + "node_modules/react-router-dom": { + "version": "7.18.1", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.18.1.tgz", + "integrity": "sha512-KaZh+X/6UtEp28x51AUYZDMg9NGoz2ja3dNHa+ta/tk40vCzKhQ/RypCWBMLbmDr6//E24Vv5uPsrqXFozdkAg==", + "license": "MIT", + "dependencies": { + "react-router": "7.18.1" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + } + }, + "node_modules/react-router/node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/rollup": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.2.tgz", + "integrity": "sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==", + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.62.2", + "@rollup/rollup-android-arm64": "4.62.2", + "@rollup/rollup-darwin-arm64": "4.62.2", + "@rollup/rollup-darwin-x64": "4.62.2", + "@rollup/rollup-freebsd-arm64": "4.62.2", + "@rollup/rollup-freebsd-x64": "4.62.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.62.2", + "@rollup/rollup-linux-arm-musleabihf": "4.62.2", + "@rollup/rollup-linux-arm64-gnu": "4.62.2", + "@rollup/rollup-linux-arm64-musl": "4.62.2", + "@rollup/rollup-linux-loong64-gnu": "4.62.2", + "@rollup/rollup-linux-loong64-musl": "4.62.2", + "@rollup/rollup-linux-ppc64-gnu": "4.62.2", + "@rollup/rollup-linux-ppc64-musl": "4.62.2", + "@rollup/rollup-linux-riscv64-gnu": "4.62.2", + "@rollup/rollup-linux-riscv64-musl": "4.62.2", + "@rollup/rollup-linux-s390x-gnu": "4.62.2", + "@rollup/rollup-linux-x64-gnu": "4.62.2", + "@rollup/rollup-linux-x64-musl": "4.62.2", + "@rollup/rollup-openbsd-x64": "4.62.2", + "@rollup/rollup-openharmony-arm64": "4.62.2", + "@rollup/rollup-win32-arm64-msvc": "4.62.2", + "@rollup/rollup-win32-ia32-msvc": "4.62.2", + "@rollup/rollup-win32-x64-gnu": "4.62.2", + "@rollup/rollup-win32-x64-msvc": "4.62.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/set-cookie-parser": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", + "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", + "license": "MIT" + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/tailwind-merge": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.6.0.tgz", + "integrity": "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/dcastil" + } + }, + "node_modules/tailwindcss": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.2.tgz", + "integrity": "sha512-WtctNNSH8A9jlMIqxzuYumOHU5uGZyRv0Q5svQl+oEPy5w84YpBxdb7MdqyiSPQge5jTJ6zFQLq0PFygdccSBA==", + "license": "MIT" + }, + "node_modules/tapable": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/tsx": { + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.0.tgz", + "integrity": "sha512-eUdUIaCr963q2h5u3+QwvYp0+eqPvn+egeqZUm0hwERCqqx1E3kK5ehbGCvqSE5MQAULr67ww0cA3jKc3YkM1w==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "devOptional": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typescript": { + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vite": { + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz", + "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", + "license": "MIT", + "peer": true, + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "license": "ISC" + } + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000000000000000000000000000000000000..36065a2898df2f55a04eef0769795b2de8d14c54 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,38 @@ +{ + "name": "react-example", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "tsx server.ts", + "build": "vite build && esbuild server.ts --bundle --platform=node --format=cjs --packages=external --sourcemap --outfile=dist/server.cjs", + "start": "node dist/server.cjs", + "clean": "rm -rf dist server.js", + "lint": "tsc --noEmit" + }, + "dependencies": { + "@google/genai": "^2.4.0", + "@tailwindcss/vite": "^4.1.14", + "@vitejs/plugin-react": "^5.0.4", + "clsx": "^2.1.1", + "dotenv": "^17.2.3", + "express": "^4.21.2", + "lucide-react": "^0.546.0", + "motion": "^12.23.24", + "react": "^19.0.1", + "react-dom": "^19.0.1", + "react-router-dom": "^7.18.1", + "tailwind-merge": "^3.6.0", + "vite": "^6.2.3" + }, + "devDependencies": { + "@types/express": "^4.17.21", + "@types/node": "^22.14.0", + "autoprefixer": "^10.4.21", + "esbuild": "^0.25.0", + "tailwindcss": "^4.1.14", + "tsx": "^4.21.0", + "typescript": "~5.8.2", + "vite": "^6.2.3" + } +} diff --git a/frontend/server.ts b/frontend/server.ts new file mode 100644 index 0000000000000000000000000000000000000000..eaa96d09fe7ac436005f9d4fa8eac05dbc094a2f --- /dev/null +++ b/frontend/server.ts @@ -0,0 +1,120 @@ +import express from 'express'; +import path from 'path'; +import { createServer as createViteServer } from 'vite'; +import crypto from 'crypto'; + +const STATUSES = [ + { status: 'QUEUED', stage: 'Initializing...' }, + { status: 'EXPANDING_PROMPT', stage: 'Enriching your prompt with AI...' }, + { status: 'GENERATING_IMAGE', stage: 'Painting the visual scene...' }, + { status: 'ESTIMATING_DEPTH', stage: 'Calculating 3D geometry...' }, + { status: 'GENERATING_AUDIO', stage: 'Composing background audio...' }, + { status: 'GENERATING_VIDEO', stage: 'Synthesizing final video...' }, + { status: 'COMPLETED', stage: 'Pipeline finished successfully!' }, +]; + +const jobs = new Map(); + +async function startServer() { + const app = express(); + const PORT = 3000; + + app.use(express.json()); + + // API Routes + app.get('/api/health', (req, res) => { + res.json({ status: 'ok', timestamp: new Date().toISOString() }); + }); + + app.post('/api/generate', (req, res) => { + const { prompt } = req.body; + const jobId = crypto.randomUUID(); + + jobs.set(jobId, { + job_id: jobId, + status: STATUSES[0].status, + stage: STATUSES[0].stage, + stepIndex: 0, + assets: {}, + error: null, + prompt + }); + + // Mock background progression + const interval = setInterval(() => { + const job = jobs.get(jobId); + if (!job) { + clearInterval(interval); + return; + } + + job.stepIndex++; + + if (job.stepIndex >= STATUSES.length - 1) { + // COMPLETED + job.status = STATUSES[STATUSES.length - 1].status; + job.stage = STATUSES[STATUSES.length - 1].stage; + + // Mock Assets + job.assets = { + image: 'https://images.unsplash.com/photo-1618005182384-a83a8bd57fbe?q=80&w=2564&auto=format&fit=crop', // beautiful abstract art + depth: 'https://images.unsplash.com/photo-1618005182384-a83a8bd57fbe?q=80&w=2564&auto=format&fit=crop&monochrome=1', // grayscale version for mock + audio: 'https://cdn.pixabay.com/download/audio/2022/10/25/audio_24db8a23d9.mp3?filename=ambient-piano-125067.mp3', // royalty free ambient + video: 'https://cdn.pixabay.com/video/2022/11/17/139369-772223838_tiny.mp4' // abstract particle video + }; + clearInterval(interval); + } else { + job.status = STATUSES[job.stepIndex].status; + job.stage = STATUSES[job.stepIndex].stage; + } + }, 4000); // 4 seconds per stage for testing + + res.json({ job_id: jobId }); + }); + + app.get('/api/status/:jobId', (req, res) => { + const job = jobs.get(req.params.jobId); + if (!job) { + return res.status(404).json({ detail: 'Job not found' }); + } + res.json({ + job_id: job.job_id, + status: job.status, + stage: job.stage + }); + }); + + app.get('/api/result/:jobId', (req, res) => { + const job = jobs.get(req.params.jobId); + if (!job) { + return res.status(404).json({ detail: 'Job not found' }); + } + res.json({ + job_id: job.job_id, + status: job.status, + assets: job.assets, + error: job.error + }); + }); + + // Vite middleware for development + if (process.env.NODE_ENV !== 'production') { + const vite = await createViteServer({ + server: { middlewareMode: true }, + appType: 'spa', + }); + app.use(vite.middlewares); + } else { + const distPath = path.join(process.cwd(), 'dist'); + app.use(express.static(distPath)); + app.get('*all', (req, res) => { + res.sendFile(path.join(distPath, 'index.html')); + }); + } + + app.listen(PORT, '0.0.0.0', () => { + console.log(`Server running on http://localhost:${PORT}`); + }); +} + +startServer(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 0000000000000000000000000000000000000000..821fba082e10f796fbaeb0f56ecb9b515fca232c --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,19 @@ +/** + * @license + * SPDX-License-Identifier: Apache-2.0 + */ + +import { BrowserRouter, Routes, Route } from 'react-router-dom'; +import Home from './pages/Home'; +import Studio from './pages/Studio'; + +export default function App() { + return ( + + + } /> + } /> + + + ); +} diff --git a/frontend/src/components/AudioPlayer.tsx b/frontend/src/components/AudioPlayer.tsx new file mode 100644 index 0000000000000000000000000000000000000000..70d58a14093d446f89245be85f736b29f02c582d --- /dev/null +++ b/frontend/src/components/AudioPlayer.tsx @@ -0,0 +1,139 @@ +import React, { useState, useRef, useEffect } from 'react'; +import { Play, Pause, Volume2, VolumeX } from 'lucide-react'; + +interface AudioPlayerProps { + src: string; +} + +export default function AudioPlayer({ src }: AudioPlayerProps) { + const [isPlaying, setIsPlaying] = useState(false); + const [duration, setDuration] = useState(0); + const [currentTime, setCurrentTime] = useState(0); + const [isMuted, setIsMuted] = useState(false); + const audioRef = useRef(null); + const progressBarRef = useRef(null); + + useEffect(() => { + setIsPlaying(false); + setCurrentTime(0); + }, [src]); + + const togglePlay = () => { + if (!audioRef.current) return; + if (isPlaying) { + audioRef.current.pause(); + } else { + audioRef.current.play(); + } + setIsPlaying(!isPlaying); + }; + + const handleTimeUpdate = () => { + if (!audioRef.current) return; + setCurrentTime(audioRef.current.currentTime); + }; + + const handleLoadedMetadata = () => { + if (!audioRef.current) return; + setDuration(audioRef.current.duration); + }; + + const handleProgressChange = (e: React.ChangeEvent) => { + if (!audioRef.current) return; + const newTime = parseFloat(e.target.value); + audioRef.current.currentTime = newTime; + setCurrentTime(newTime); + }; + + const toggleMute = () => { + if (!audioRef.current) return; + audioRef.current.muted = !isMuted; + setIsMuted(!isMuted); + }; + + const formatTime = (time: number) => { + if (isNaN(time)) return '0:00'; + const minutes = Math.floor(time / 60); + const seconds = Math.floor(time % 60); + return `${minutes}:${seconds < 10 ? '0' : ''}${seconds}`; + }; + + return ( +
+
+ ); +} diff --git a/frontend/src/components/DepthSlider.tsx b/frontend/src/components/DepthSlider.tsx new file mode 100644 index 0000000000000000000000000000000000000000..edebd6ad4c5222f99b4d415ba0eb7d507a1c8c84 --- /dev/null +++ b/frontend/src/components/DepthSlider.tsx @@ -0,0 +1,99 @@ +import React, { useState, useRef } from 'react'; + +interface DepthSliderProps { + image: string; + depth: string; +} + +export default function DepthSlider({ image, depth }: DepthSliderProps) { + const [sliderPosition, setSliderPosition] = useState(50); // percentage (0 - 100) + const containerRef = useRef(null); + const isDragging = useRef(false); + + const handleMove = (clientX: number) => { + if (!containerRef.current) return; + const rect = containerRef.current.getBoundingClientRect(); + const x = clientX - rect.left; + const percentage = Math.max(0, Math.min(100, (x / rect.width) * 100)); + setSliderPosition(percentage); + }; + + const handleMouseMove = (e: React.MouseEvent) => { + // We update on hover or active dragging + if (e.buttons === 1 || isDragging.current) { + handleMove(e.clientX); + } + }; + + const handleMouseDown = (e: React.MouseEvent) => { + isDragging.current = true; + handleMove(e.clientX); + }; + + const handleMouseUp = () => { + isDragging.current = false; + }; + + const handleTouchMove = (e: React.TouchEvent) => { + if (e.touches[0]) { + handleMove(e.touches[0].clientX); + } + }; + + return ( +
{ isDragging.current = true; }} + onTouchEnd={() => { isDragging.current = false; }} + className="relative w-full aspect-square rounded-2xl overflow-hidden border border-white/10 select-none cursor-ew-resize bg-gray-950 shadow-inner group animate-in fade-in zoom-in-95 duration-500" + > + {/* Base Image (Underneath) */} + Base visual + + {/* Depth Map (On Top, clipped to reveal right side) */} +
+ Depth map visual +
+ + {/* Divider Line */} +
+ {/* Slider Handle Knob */} +
+
+ + +
+
+
+ + {/* Floating Labels */} +
+ Original Image +
+
+ Depth Geometry +
+
+ ); +} diff --git a/frontend/src/components/VideoPlayer.tsx b/frontend/src/components/VideoPlayer.tsx new file mode 100644 index 0000000000000000000000000000000000000000..d15d6e724ad5558c90bd0749db535ec9780b485b --- /dev/null +++ b/frontend/src/components/VideoPlayer.tsx @@ -0,0 +1,101 @@ +import React, { useRef, useState, useEffect } from 'react'; +import { Play, Pause, RefreshCw, Volume2, VolumeX, Maximize } from 'lucide-react'; + +interface VideoPlayerProps { + src: string; +} + +export default function VideoPlayer({ src }: VideoPlayerProps) { + const [isPlaying, setIsPlaying] = useState(false); + const [progress, setProgress] = useState(0); + const [isMuted, setIsMuted] = useState(false); + const videoRef = useRef(null); + + useEffect(() => { + setIsPlaying(false); + setProgress(0); + }, [src]); + + const togglePlay = () => { + if (!videoRef.current) return; + if (isPlaying) { + videoRef.current.pause(); + } else { + videoRef.current.play(); + } + setIsPlaying(!isPlaying); + }; + + const handleTimeUpdate = () => { + if (!videoRef.current) return; + const percentage = (videoRef.current.currentTime / videoRef.current.duration) * 100; + setProgress(percentage); + }; + + const toggleMute = () => { + if (!videoRef.current) return; + videoRef.current.muted = !isMuted; + setIsMuted(!isMuted); + }; + + const handleFullscreen = () => { + if (!videoRef.current) return; + if (videoRef.current.requestFullscreen) { + videoRef.current.requestFullscreen(); + } + }; + + return ( +
+