Spaces:
Build error
Build error
suryanshp1 commited on
Commit ·
c4b8bb2
1
Parent(s): 5ab7b68
feat: update .gitignore and remove outdated design, requirements, and tasks documents for enhanced guardrails system
Browse files
.gitignore
CHANGED
|
@@ -1,2 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
.env
|
| 2 |
-
venv
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Byte-compiled / optimized / DLL files
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*$py.class
|
| 5 |
+
|
| 6 |
+
# C extensions
|
| 7 |
+
*.so
|
| 8 |
+
|
| 9 |
+
# Distribution / packaging
|
| 10 |
+
.Python
|
| 11 |
+
build/
|
| 12 |
+
develop-eggs/
|
| 13 |
+
dist/
|
| 14 |
+
downloads/
|
| 15 |
+
eggs/
|
| 16 |
+
.eggs/
|
| 17 |
+
lib/
|
| 18 |
+
lib64/
|
| 19 |
+
parts/
|
| 20 |
+
sdist/
|
| 21 |
+
var/
|
| 22 |
+
wheels/
|
| 23 |
+
share/python-wheels/
|
| 24 |
+
*.egg-info/
|
| 25 |
+
.installed.cfg
|
| 26 |
+
*.egg
|
| 27 |
+
MANIFEST
|
| 28 |
+
|
| 29 |
+
# PyInstaller
|
| 30 |
+
# Usually these files are written by a python script from a template
|
| 31 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
| 32 |
+
*.manifest
|
| 33 |
+
*.spec
|
| 34 |
+
|
| 35 |
+
# Installer logs
|
| 36 |
+
pip-log.txt
|
| 37 |
+
pip-delete-this-directory.txt
|
| 38 |
+
|
| 39 |
+
# Unit test / coverage reports
|
| 40 |
+
htmlcov/
|
| 41 |
+
.tox/
|
| 42 |
+
.nox/
|
| 43 |
+
.coverage
|
| 44 |
+
.coverage.*
|
| 45 |
+
.cache
|
| 46 |
+
nosetests.xml
|
| 47 |
+
coverage.xml
|
| 48 |
+
*.cover
|
| 49 |
+
*.py,cover
|
| 50 |
+
.hypothesis/
|
| 51 |
+
.pytest_cache/
|
| 52 |
+
cover/
|
| 53 |
+
|
| 54 |
+
# Translations
|
| 55 |
+
*.mo
|
| 56 |
+
*.pot
|
| 57 |
+
|
| 58 |
+
# Django stuff:
|
| 59 |
+
*.log
|
| 60 |
+
local_settings.py
|
| 61 |
+
db.sqlite3
|
| 62 |
+
db.sqlite3-journal
|
| 63 |
+
|
| 64 |
+
# Flask stuff:
|
| 65 |
+
instance/
|
| 66 |
+
.webassets-cache
|
| 67 |
+
|
| 68 |
+
# Scrapy stuff:
|
| 69 |
+
.scrapy
|
| 70 |
+
|
| 71 |
+
# Sphinx documentation
|
| 72 |
+
docs/_build/
|
| 73 |
+
|
| 74 |
+
# PyBuilder
|
| 75 |
+
.pybuilder/
|
| 76 |
+
target/
|
| 77 |
+
|
| 78 |
+
# Jupyter Notebook
|
| 79 |
+
.ipynb_checkpoints
|
| 80 |
+
|
| 81 |
+
# IPython
|
| 82 |
+
profile_default/
|
| 83 |
+
ipython_config.py
|
| 84 |
+
|
| 85 |
+
# pyenv
|
| 86 |
+
# For a library or package, you might want to ignore these files since the code is
|
| 87 |
+
# intended to run in multiple environments; otherwise, check them in:
|
| 88 |
+
# .python-version
|
| 89 |
+
|
| 90 |
+
# pipenv
|
| 91 |
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
| 92 |
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
| 93 |
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
| 94 |
+
# install all needed dependencies.
|
| 95 |
+
#Pipfile.lock
|
| 96 |
+
|
| 97 |
+
# poetry
|
| 98 |
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
| 99 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
| 100 |
+
# commonly ignored for libraries.
|
| 101 |
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
| 102 |
+
#poetry.lock
|
| 103 |
+
|
| 104 |
+
# pdm
|
| 105 |
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
| 106 |
+
#pdm.lock
|
| 107 |
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
| 108 |
+
# in version control.
|
| 109 |
+
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
| 110 |
+
.pdm.toml
|
| 111 |
+
.pdm-python
|
| 112 |
+
.pdm-build/
|
| 113 |
+
|
| 114 |
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
| 115 |
+
__pypackages__/
|
| 116 |
+
|
| 117 |
+
# Celery stuff
|
| 118 |
+
celerybeat-schedule
|
| 119 |
+
celerybeat.pid
|
| 120 |
+
|
| 121 |
+
# SageMath parsed files
|
| 122 |
+
*.sage.py
|
| 123 |
+
|
| 124 |
+
# Environments
|
| 125 |
.env
|
| 126 |
+
.venv
|
| 127 |
+
env/
|
| 128 |
+
venv/
|
| 129 |
+
ENV/
|
| 130 |
+
env.bak/
|
| 131 |
+
venv.bak/
|
| 132 |
+
|
| 133 |
+
# Spyder project settings
|
| 134 |
+
.spyderproject
|
| 135 |
+
.spyproject
|
| 136 |
+
|
| 137 |
+
# Rope project settings
|
| 138 |
+
.ropeproject
|
| 139 |
+
|
| 140 |
+
# mkdocs documentation
|
| 141 |
+
/site
|
| 142 |
+
|
| 143 |
+
# mypy
|
| 144 |
+
.mypy_cache/
|
| 145 |
+
.dmypy.json
|
| 146 |
+
dmypy.json
|
| 147 |
+
|
| 148 |
+
# Pyre type checker
|
| 149 |
+
.pyre/
|
| 150 |
+
|
| 151 |
+
# pytype static type analyzer
|
| 152 |
+
.pytype/
|
| 153 |
+
|
| 154 |
+
# Cython debug symbols
|
| 155 |
+
cython_debug/
|
| 156 |
+
|
| 157 |
+
# PyCharm
|
| 158 |
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
| 159 |
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
| 160 |
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
| 161 |
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
| 162 |
+
#.idea/
|
.kiro/specs/enhanced-guardrails-system/design.md
DELETED
|
@@ -1,330 +0,0 @@
|
|
| 1 |
-
# Design Document
|
| 2 |
-
|
| 3 |
-
## Overview
|
| 4 |
-
|
| 5 |
-
The Enhanced Guardrails System extends the existing guardrails functionality by integrating Guardrails Hub validators, implementing comprehensive input/output guards, and adding support for structured data generation. The design maintains backward compatibility while adding robust validation capabilities that can operate both embedded and as a standalone service.
|
| 6 |
-
|
| 7 |
-
## Architecture
|
| 8 |
-
|
| 9 |
-
### High-Level Architecture
|
| 10 |
-
|
| 11 |
-
```mermaid
|
| 12 |
-
graph TB
|
| 13 |
-
subgraph "Client Layer"
|
| 14 |
-
UI[Streamlit UI]
|
| 15 |
-
API[REST API]
|
| 16 |
-
end
|
| 17 |
-
|
| 18 |
-
subgraph "Guardrails Layer"
|
| 19 |
-
GS[Guardrails Service]
|
| 20 |
-
GC[Guard Coordinator]
|
| 21 |
-
HI[Hub Integration]
|
| 22 |
-
end
|
| 23 |
-
|
| 24 |
-
subgraph "Validation Guards"
|
| 25 |
-
IG[Input Guards]
|
| 26 |
-
OG[Output Guards]
|
| 27 |
-
SG[Structured Guards]
|
| 28 |
-
CG[Content Guards]
|
| 29 |
-
end
|
| 30 |
-
|
| 31 |
-
subgraph "LLM Layer"
|
| 32 |
-
LW[LLM Wrapper]
|
| 33 |
-
LLM[Language Models]
|
| 34 |
-
end
|
| 35 |
-
|
| 36 |
-
subgraph "External Services"
|
| 37 |
-
GH[Guardrails Hub]
|
| 38 |
-
FS[Flask Server]
|
| 39 |
-
end
|
| 40 |
-
|
| 41 |
-
UI --> GS
|
| 42 |
-
API --> GS
|
| 43 |
-
GS --> GC
|
| 44 |
-
GC --> IG
|
| 45 |
-
GC --> OG
|
| 46 |
-
GC --> SG
|
| 47 |
-
GC --> CG
|
| 48 |
-
GC --> LW
|
| 49 |
-
LW --> LLM
|
| 50 |
-
HI --> GH
|
| 51 |
-
GS --> FS
|
| 52 |
-
```
|
| 53 |
-
|
| 54 |
-
### Component Interaction Flow
|
| 55 |
-
|
| 56 |
-
```mermaid
|
| 57 |
-
sequenceDiagram
|
| 58 |
-
participant User
|
| 59 |
-
participant UI
|
| 60 |
-
participant GuardService
|
| 61 |
-
participant InputGuards
|
| 62 |
-
participant LLMWrapper
|
| 63 |
-
participant OutputGuards
|
| 64 |
-
participant LLM
|
| 65 |
-
|
| 66 |
-
User->>UI: Submit Input
|
| 67 |
-
UI->>GuardService: Validate Request
|
| 68 |
-
GuardService->>InputGuards: Apply Input Validation
|
| 69 |
-
InputGuards-->>GuardService: Validation Result
|
| 70 |
-
|
| 71 |
-
alt Input Valid
|
| 72 |
-
GuardService->>LLMWrapper: Process Request
|
| 73 |
-
LLMWrapper->>LLM: Generate Response
|
| 74 |
-
LLM-->>LLMWrapper: Raw Response
|
| 75 |
-
LLMWrapper->>OutputGuards: Validate Output
|
| 76 |
-
OutputGuards-->>LLMWrapper: Validated Output
|
| 77 |
-
LLMWrapper-->>GuardService: Final Response
|
| 78 |
-
GuardService-->>UI: Safe Response
|
| 79 |
-
UI-->>User: Display Result
|
| 80 |
-
else Input Invalid
|
| 81 |
-
GuardService-->>UI: Validation Error
|
| 82 |
-
UI-->>User: Error Message
|
| 83 |
-
end
|
| 84 |
-
```
|
| 85 |
-
|
| 86 |
-
## Components and Interfaces
|
| 87 |
-
|
| 88 |
-
### 1. Enhanced Guardrails Configuration
|
| 89 |
-
|
| 90 |
-
**Purpose:** Manages Guardrails Hub integration and guard configurations
|
| 91 |
-
|
| 92 |
-
**Key Interfaces:**
|
| 93 |
-
```python
|
| 94 |
-
class EnhancedGuardrailsConfig:
|
| 95 |
-
def install_hub_validator(self, validator_name: str) -> bool
|
| 96 |
-
def create_guard_from_hub(self, validators: List[str], config: Dict) -> Guard
|
| 97 |
-
def get_structured_guard(self, model_class: BaseModel) -> Guard
|
| 98 |
-
def configure_server_mode(self, host: str, port: int) -> bool
|
| 99 |
-
```
|
| 100 |
-
|
| 101 |
-
**Responsibilities:**
|
| 102 |
-
- Manage Guardrails Hub CLI integration
|
| 103 |
-
- Cache installed validators
|
| 104 |
-
- Configure guard combinations
|
| 105 |
-
- Handle server mode setup
|
| 106 |
-
|
| 107 |
-
### 2. Guard Factory
|
| 108 |
-
|
| 109 |
-
**Purpose:** Creates and manages different types of guards based on use cases
|
| 110 |
-
|
| 111 |
-
**Key Interfaces:**
|
| 112 |
-
```python
|
| 113 |
-
class GuardFactory:
|
| 114 |
-
def create_input_guard(self, validators: List[str]) -> Guard
|
| 115 |
-
def create_output_guard(self, quality_rules: Dict) -> Guard
|
| 116 |
-
def create_structured_guard(self, schema: BaseModel) -> Guard
|
| 117 |
-
def create_composite_guard(self, guard_configs: List[Dict]) -> Guard
|
| 118 |
-
```
|
| 119 |
-
|
| 120 |
-
**Responsibilities:**
|
| 121 |
-
- Instantiate guards with proper configurations
|
| 122 |
-
- Manage guard lifecycle
|
| 123 |
-
- Handle guard composition and chaining
|
| 124 |
-
|
| 125 |
-
### 3. Validation Orchestrator
|
| 126 |
-
|
| 127 |
-
**Purpose:** Coordinates validation across multiple guards and handles failures
|
| 128 |
-
|
| 129 |
-
**Key Interfaces:**
|
| 130 |
-
```python
|
| 131 |
-
class ValidationOrchestrator:
|
| 132 |
-
def validate_input(self, content: str, context: Dict) -> ValidationResult
|
| 133 |
-
def validate_output(self, content: str, schema: Optional[BaseModel]) -> ValidationResult
|
| 134 |
-
def apply_corrections(self, content: str, failures: List[ValidationFailure]) -> str
|
| 135 |
-
def get_validation_metrics(self) -> Dict[str, Any]
|
| 136 |
-
```
|
| 137 |
-
|
| 138 |
-
**Responsibilities:**
|
| 139 |
-
- Execute validation pipelines
|
| 140 |
-
- Handle validation failures and retries
|
| 141 |
-
- Apply automatic corrections
|
| 142 |
-
- Collect validation metrics
|
| 143 |
-
|
| 144 |
-
### 4. Hub Integration Manager
|
| 145 |
-
|
| 146 |
-
**Purpose:** Manages integration with Guardrails Hub for validator installation and updates
|
| 147 |
-
|
| 148 |
-
**Key Interfaces:**
|
| 149 |
-
```python
|
| 150 |
-
class HubIntegrationManager:
|
| 151 |
-
def configure_hub_cli(self) -> bool
|
| 152 |
-
def install_validator(self, hub_path: str) -> bool
|
| 153 |
-
def list_available_validators(self) -> List[str]
|
| 154 |
-
def update_validators(self) -> Dict[str, bool]
|
| 155 |
-
```
|
| 156 |
-
|
| 157 |
-
**Responsibilities:**
|
| 158 |
-
- Configure Guardrails Hub CLI
|
| 159 |
-
- Install and update validators
|
| 160 |
-
- Manage validator dependencies
|
| 161 |
-
- Handle offline fallbacks
|
| 162 |
-
|
| 163 |
-
### 5. Structured Data Generator
|
| 164 |
-
|
| 165 |
-
**Purpose:** Generates structured data from LLMs using Pydantic models and Guardrails
|
| 166 |
-
|
| 167 |
-
**Key Interfaces:**
|
| 168 |
-
```python
|
| 169 |
-
class StructuredDataGenerator:
|
| 170 |
-
def generate_with_schema(self, prompt: str, schema: BaseModel, llm: BaseLanguageModel) -> Any
|
| 171 |
-
def validate_structured_output(self, data: Any, schema: BaseModel) -> ValidationResult
|
| 172 |
-
def handle_function_calling(self, llm: BaseLanguageModel, schema: BaseModel) -> bool
|
| 173 |
-
def optimize_prompt_for_schema(self, prompt: str, schema: BaseModel) -> str
|
| 174 |
-
```
|
| 175 |
-
|
| 176 |
-
**Responsibilities:**
|
| 177 |
-
- Generate structured data using function calling or prompt optimization
|
| 178 |
-
- Validate structured outputs against Pydantic schemas
|
| 179 |
-
- Handle LLM compatibility differences
|
| 180 |
-
- Retry failed structured generation
|
| 181 |
-
|
| 182 |
-
### 6. Guardrails Server Interface
|
| 183 |
-
|
| 184 |
-
**Purpose:** Provides REST API interface for standalone Guardrails service
|
| 185 |
-
|
| 186 |
-
**Key Interfaces:**
|
| 187 |
-
```python
|
| 188 |
-
class GuardrailsServerInterface:
|
| 189 |
-
def start_server(self, config_path: str, host: str, port: int) -> bool
|
| 190 |
-
def validate_via_api(self, content: str, guard_name: str) -> ValidationResult
|
| 191 |
-
def proxy_openai_request(self, request: Dict) -> Dict
|
| 192 |
-
def health_check(self) -> bool
|
| 193 |
-
```
|
| 194 |
-
|
| 195 |
-
**Responsibilities:**
|
| 196 |
-
- Start and manage Flask server
|
| 197 |
-
- Provide REST API endpoints
|
| 198 |
-
- Handle OpenAI SDK proxy functionality
|
| 199 |
-
- Monitor server health
|
| 200 |
-
|
| 201 |
-
## Data Models
|
| 202 |
-
|
| 203 |
-
### Validation Result Model
|
| 204 |
-
```python
|
| 205 |
-
class ValidationResult(BaseModel):
|
| 206 |
-
is_valid: bool
|
| 207 |
-
processed_content: str
|
| 208 |
-
original_content: str
|
| 209 |
-
failures: List[ValidationFailure]
|
| 210 |
-
corrections_applied: List[str]
|
| 211 |
-
validation_time: float
|
| 212 |
-
guard_name: str
|
| 213 |
-
```
|
| 214 |
-
|
| 215 |
-
### Guard Configuration Model
|
| 216 |
-
```python
|
| 217 |
-
class GuardConfig(BaseModel):
|
| 218 |
-
name: str
|
| 219 |
-
validators: List[ValidatorConfig]
|
| 220 |
-
failure_action: OnFailAction
|
| 221 |
-
retry_count: int = 3
|
| 222 |
-
timeout: float = 30.0
|
| 223 |
-
enabled: bool = True
|
| 224 |
-
```
|
| 225 |
-
|
| 226 |
-
### Validator Configuration Model
|
| 227 |
-
```python
|
| 228 |
-
class ValidatorConfig(BaseModel):
|
| 229 |
-
type: str # "hub", "builtin", "custom"
|
| 230 |
-
name: str
|
| 231 |
-
parameters: Dict[str, Any]
|
| 232 |
-
on_fail: OnFailAction
|
| 233 |
-
priority: int = 0
|
| 234 |
-
```
|
| 235 |
-
|
| 236 |
-
### Structured Schema Registry
|
| 237 |
-
```python
|
| 238 |
-
class SchemaRegistry:
|
| 239 |
-
def register_schema(self, name: str, schema: BaseModel) -> None
|
| 240 |
-
def get_schema(self, name: str) -> Optional[BaseModel]
|
| 241 |
-
def list_schemas(self) -> List[str]
|
| 242 |
-
```
|
| 243 |
-
|
| 244 |
-
## Error Handling
|
| 245 |
-
|
| 246 |
-
### Validation Failure Handling
|
| 247 |
-
1. **Input Validation Failures:**
|
| 248 |
-
- Log failure details with context
|
| 249 |
-
- Provide user-friendly error messages
|
| 250 |
-
- Offer suggestions for correction
|
| 251 |
-
- Block processing until valid input
|
| 252 |
-
|
| 253 |
-
2. **Output Validation Failures:**
|
| 254 |
-
- Apply automatic corrections when possible
|
| 255 |
-
- Regenerate response with modified prompts
|
| 256 |
-
- Fallback to safe default responses
|
| 257 |
-
- Log incidents for analysis
|
| 258 |
-
|
| 259 |
-
3. **Hub Integration Failures:**
|
| 260 |
-
- Cache validators locally for offline use
|
| 261 |
-
- Fallback to built-in validators
|
| 262 |
-
- Retry with exponential backoff
|
| 263 |
-
- Graceful degradation of functionality
|
| 264 |
-
|
| 265 |
-
4. **Server Mode Failures:**
|
| 266 |
-
- Automatic fallback to embedded mode
|
| 267 |
-
- Health check monitoring
|
| 268 |
-
- Circuit breaker pattern for resilience
|
| 269 |
-
- Connection pooling and retry logic
|
| 270 |
-
|
| 271 |
-
### Error Recovery Strategies
|
| 272 |
-
```python
|
| 273 |
-
class ErrorRecoveryStrategy:
|
| 274 |
-
def handle_validation_timeout(self, context: Dict) -> ValidationResult
|
| 275 |
-
def handle_hub_unavailable(self, validator_name: str) -> Guard
|
| 276 |
-
def handle_server_unreachable(self) -> None
|
| 277 |
-
def handle_schema_validation_error(self, data: Any, schema: BaseModel) -> Any
|
| 278 |
-
```
|
| 279 |
-
|
| 280 |
-
## Testing Strategy
|
| 281 |
-
|
| 282 |
-
### Unit Testing
|
| 283 |
-
- **Guard Factory Tests:** Verify correct guard instantiation and configuration
|
| 284 |
-
- **Validation Logic Tests:** Test individual validator behavior and edge cases
|
| 285 |
-
- **Hub Integration Tests:** Mock Hub API responses and test error handling
|
| 286 |
-
- **Structured Generation Tests:** Validate schema compliance and data integrity
|
| 287 |
-
|
| 288 |
-
### Integration Testing
|
| 289 |
-
- **End-to-End Validation Flow:** Test complete input → processing → output pipeline
|
| 290 |
-
- **Server Mode Integration:** Test REST API functionality and OpenAI proxy
|
| 291 |
-
- **LLM Wrapper Integration:** Verify seamless integration with existing LLM calls
|
| 292 |
-
- **Fallback Mechanism Tests:** Test graceful degradation scenarios
|
| 293 |
-
|
| 294 |
-
### Performance Testing
|
| 295 |
-
- **Validation Latency:** Measure impact on response times
|
| 296 |
-
- **Concurrent Validation:** Test multiple simultaneous validation requests
|
| 297 |
-
- **Memory Usage:** Monitor memory consumption with large guard configurations
|
| 298 |
-
- **Server Scalability:** Test Guardrails server under load
|
| 299 |
-
|
| 300 |
-
### Security Testing
|
| 301 |
-
- **Input Sanitization:** Test malicious input handling
|
| 302 |
-
- **Output Filtering:** Verify sensitive content detection and removal
|
| 303 |
-
- **Configuration Security:** Test secure storage of guard configurations
|
| 304 |
-
- **API Security:** Validate REST API authentication and authorization
|
| 305 |
-
|
| 306 |
-
## Implementation Phases
|
| 307 |
-
|
| 308 |
-
### Phase 1: Hub Integration Foundation
|
| 309 |
-
- Configure Guardrails Hub CLI integration
|
| 310 |
-
- Implement validator installation and caching
|
| 311 |
-
- Create basic guard factory with hub validators
|
| 312 |
-
- Add configuration management for hub validators
|
| 313 |
-
|
| 314 |
-
### Phase 2: Enhanced Input/Output Guards
|
| 315 |
-
- Extend existing input validation with hub validators
|
| 316 |
-
- Implement comprehensive output quality guards
|
| 317 |
-
- Add structured data generation capabilities
|
| 318 |
-
- Integrate with existing LLM wrapper
|
| 319 |
-
|
| 320 |
-
### Phase 3: Server Mode Implementation
|
| 321 |
-
- Implement Guardrails Flask server
|
| 322 |
-
- Add REST API endpoints for validation
|
| 323 |
-
- Create OpenAI SDK proxy functionality
|
| 324 |
-
- Implement health monitoring and fallback
|
| 325 |
-
|
| 326 |
-
### Phase 4: Advanced Features and Optimization
|
| 327 |
-
- Add validation metrics and monitoring
|
| 328 |
-
- Implement advanced error recovery
|
| 329 |
-
- Optimize performance for production use
|
| 330 |
-
- Add comprehensive logging and debugging
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.kiro/specs/enhanced-guardrails-system/requirements.md
DELETED
|
@@ -1,84 +0,0 @@
|
|
| 1 |
-
# Requirements Document
|
| 2 |
-
|
| 3 |
-
## Introduction
|
| 4 |
-
|
| 5 |
-
This feature enhances the existing guardrails system in the LangGraph Agentic AI application by integrating comprehensive input and output guards using the Guardrails Hub. The enhancement will provide robust validation, safety measures, and structured data generation capabilities for LLM interactions, ensuring reliable and safe AI responses while maintaining the existing application architecture.
|
| 6 |
-
|
| 7 |
-
## Requirements
|
| 8 |
-
|
| 9 |
-
### Requirement 1
|
| 10 |
-
|
| 11 |
-
**User Story:** As a developer, I want to configure and use Guardrails Hub validators so that I can leverage pre-built validation rules for common use cases.
|
| 12 |
-
|
| 13 |
-
#### Acceptance Criteria
|
| 14 |
-
|
| 15 |
-
1. WHEN the application starts THEN the system SHALL automatically configure Guardrails Hub CLI integration
|
| 16 |
-
2. WHEN installing guardrails from the hub THEN the system SHALL support regex matching, competitor checking, and toxic language detection
|
| 17 |
-
3. WHEN a guardrail is installed THEN the system SHALL cache the configuration for reuse across sessions
|
| 18 |
-
4. IF Guardrails Hub is unavailable THEN the system SHALL gracefully fallback to basic validation without breaking functionality
|
| 19 |
-
|
| 20 |
-
### Requirement 2
|
| 21 |
-
|
| 22 |
-
**User Story:** As a system administrator, I want comprehensive input validation guards so that malicious or inappropriate content is filtered before reaching the LLM.
|
| 23 |
-
|
| 24 |
-
#### Acceptance Criteria
|
| 25 |
-
|
| 26 |
-
1. WHEN user input is received THEN the system SHALL validate against regex patterns, toxic language, and profanity
|
| 27 |
-
2. WHEN input validation fails THEN the system SHALL provide clear error messages and prevent LLM processing
|
| 28 |
-
3. WHEN competitor names are detected THEN the system SHALL either block or sanitize the content based on configuration
|
| 29 |
-
4. IF input exceeds length limits THEN the system SHALL reject the input with appropriate feedback
|
| 30 |
-
|
| 31 |
-
### Requirement 3
|
| 32 |
-
|
| 33 |
-
**User Story:** As a developer, I want output validation guards so that LLM responses meet quality and safety standards before being returned to users.
|
| 34 |
-
|
| 35 |
-
#### Acceptance Criteria
|
| 36 |
-
|
| 37 |
-
1. WHEN LLM generates a response THEN the system SHALL validate output for toxic content, length, and readability
|
| 38 |
-
2. WHEN output validation fails THEN the system SHALL either regenerate the response or apply corrective measures
|
| 39 |
-
3. WHEN structured data is requested THEN the system SHALL enforce Pydantic model validation
|
| 40 |
-
4. IF output contains sensitive information THEN the system SHALL sanitize or block the response
|
| 41 |
-
|
| 42 |
-
### Requirement 4
|
| 43 |
-
|
| 44 |
-
**User Story:** As a developer, I want to generate structured data from LLMs using Guardrails so that I can ensure consistent output formats for downstream processing.
|
| 45 |
-
|
| 46 |
-
#### Acceptance Criteria
|
| 47 |
-
|
| 48 |
-
1. WHEN requesting structured output THEN the system SHALL use Pydantic models to define expected schemas
|
| 49 |
-
2. WHEN LLM supports function calling THEN the system SHALL use function call syntax for structured generation
|
| 50 |
-
3. WHEN LLM doesn't support function calling THEN the system SHALL use prompt optimization with schema injection
|
| 51 |
-
4. IF structured validation fails THEN the system SHALL retry with corrected prompts up to a configurable limit
|
| 52 |
-
|
| 53 |
-
### Requirement 5
|
| 54 |
-
|
| 55 |
-
**User Story:** As a system operator, I want Guardrails to run as a standalone service so that I can scale validation independently and integrate with multiple applications.
|
| 56 |
-
|
| 57 |
-
#### Acceptance Criteria
|
| 58 |
-
|
| 59 |
-
1. WHEN deploying the application THEN the system SHALL support running Guardrails as a Flask service
|
| 60 |
-
2. WHEN using Guardrails server THEN the system SHALL provide REST API endpoints for validation
|
| 61 |
-
3. WHEN integrating with OpenAI SDK THEN the system SHALL support transparent proxy functionality
|
| 62 |
-
4. IF the Guardrails server is unavailable THEN the system SHALL fallback to embedded validation
|
| 63 |
-
|
| 64 |
-
### Requirement 6
|
| 65 |
-
|
| 66 |
-
**User Story:** As a developer, I want configurable guard combinations so that I can apply multiple validation rules simultaneously based on use case requirements.
|
| 67 |
-
|
| 68 |
-
#### Acceptance Criteria
|
| 69 |
-
|
| 70 |
-
1. WHEN configuring guards THEN the system SHALL support chaining multiple validators with different failure actions
|
| 71 |
-
2. WHEN validation fails THEN the system SHALL provide detailed feedback about which specific guards failed
|
| 72 |
-
3. WHEN guards conflict THEN the system SHALL apply a configurable precedence order
|
| 73 |
-
4. IF performance is impacted THEN the system SHALL support async validation for non-blocking operations
|
| 74 |
-
|
| 75 |
-
### Requirement 7
|
| 76 |
-
|
| 77 |
-
**User Story:** As a system administrator, I want comprehensive monitoring and logging of guard activities so that I can track validation effectiveness and troubleshoot issues.
|
| 78 |
-
|
| 79 |
-
#### Acceptance Criteria
|
| 80 |
-
|
| 81 |
-
1. WHEN guards are executed THEN the system SHALL log validation attempts, results, and performance metrics
|
| 82 |
-
2. WHEN validation fails THEN the system SHALL capture detailed error information for analysis
|
| 83 |
-
3. WHEN guards are bypassed THEN the system SHALL log the bypass reason and context
|
| 84 |
-
4. IF monitoring data exceeds storage limits THEN the system SHALL implement configurable retention policies
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.kiro/specs/enhanced-guardrails-system/tasks.md
DELETED
|
@@ -1,95 +0,0 @@
|
|
| 1 |
-
# Implementation Plan
|
| 2 |
-
|
| 3 |
-
- [ ] 1. Set up Guardrails Hub integration foundation
|
| 4 |
-
- Update requirements.txt with additional Guardrails Hub dependencies
|
| 5 |
-
- Create hub integration manager class with CLI configuration methods
|
| 6 |
-
- Implement validator installation and caching functionality
|
| 7 |
-
- Add error handling for hub connectivity issues
|
| 8 |
-
- _Requirements: 1.1, 1.2, 1.3, 1.4_
|
| 9 |
-
|
| 10 |
-
- [ ] 2. Create enhanced guard factory and configuration system
|
| 11 |
-
- Extend GuardrailsConfig class to support hub validators
|
| 12 |
-
- Implement GuardFactory class for creating different guard types
|
| 13 |
-
- Add methods for installing regex_match, competitor_check, and toxic_language validators
|
| 14 |
-
- Create configuration caching system for installed validators
|
| 15 |
-
- _Requirements: 1.1, 1.2, 6.1, 6.2_
|
| 16 |
-
|
| 17 |
-
- [ ] 3. Implement comprehensive input validation guards
|
| 18 |
-
- Create enhanced input validation methods using hub validators
|
| 19 |
-
- Add regex pattern validation for phone numbers and other formats
|
| 20 |
-
- Implement competitor name detection and filtering
|
| 21 |
-
- Add configurable input length and content validation
|
| 22 |
-
- Write unit tests for input validation scenarios
|
| 23 |
-
- _Requirements: 2.1, 2.2, 2.3, 2.4_
|
| 24 |
-
|
| 25 |
-
- [ ] 4. Develop advanced output validation and quality guards
|
| 26 |
-
- Implement output quality validation using hub validators
|
| 27 |
-
- Add toxic language detection for LLM responses
|
| 28 |
-
- Create output length and readability validation
|
| 29 |
-
- Implement sensitive information detection and sanitization
|
| 30 |
-
- Write unit tests for output validation scenarios
|
| 31 |
-
- _Requirements: 3.1, 3.2, 3.4_
|
| 32 |
-
|
| 33 |
-
- [ ] 5. Create structured data generation system
|
| 34 |
-
- Implement StructuredDataGenerator class with Pydantic model support
|
| 35 |
-
- Add function calling detection and implementation for compatible LLMs
|
| 36 |
-
- Create prompt optimization for non-function-calling LLMs
|
| 37 |
-
- Implement schema validation and retry logic for failed generations
|
| 38 |
-
- Write unit tests for structured data generation
|
| 39 |
-
- _Requirements: 4.1, 4.2, 4.3, 4.4_
|
| 40 |
-
|
| 41 |
-
- [ ] 6. Implement validation orchestrator and error handling
|
| 42 |
-
- Create ValidationOrchestrator class to coordinate multiple guards
|
| 43 |
-
- Implement guard chaining and precedence handling
|
| 44 |
-
- Add automatic correction and retry mechanisms
|
| 45 |
-
- Create comprehensive error recovery strategies
|
| 46 |
-
- Write unit tests for orchestration scenarios
|
| 47 |
-
- _Requirements: 6.1, 6.2, 6.3_
|
| 48 |
-
|
| 49 |
-
- [ ] 7. Build Guardrails server interface and REST API
|
| 50 |
-
- Create GuardrailsServerInterface class for Flask server management
|
| 51 |
-
- Implement REST API endpoints for validation services
|
| 52 |
-
- Add OpenAI SDK proxy functionality for transparent integration
|
| 53 |
-
- Implement health check and monitoring endpoints
|
| 54 |
-
- Write integration tests for server functionality
|
| 55 |
-
- _Requirements: 5.1, 5.2, 5.3, 5.4_
|
| 56 |
-
|
| 57 |
-
- [ ] 8. Enhance existing LLM wrapper with new capabilities
|
| 58 |
-
- Update GuardrailsLLMWrapper to use new validation orchestrator
|
| 59 |
-
- Add support for structured data generation in LLM calls
|
| 60 |
-
- Implement fallback mechanisms for validation failures
|
| 61 |
-
- Add async validation support for improved performance
|
| 62 |
-
- Write integration tests for enhanced LLM wrapper
|
| 63 |
-
- _Requirements: 3.1, 3.2, 4.1, 6.4_
|
| 64 |
-
|
| 65 |
-
- [ ] 9. Add comprehensive monitoring and logging system
|
| 66 |
-
- Implement validation metrics collection and reporting
|
| 67 |
-
- Add detailed logging for guard execution and failures
|
| 68 |
-
- Create performance monitoring for validation latency
|
| 69 |
-
- Implement configurable retention policies for monitoring data
|
| 70 |
-
- Write unit tests for monitoring functionality
|
| 71 |
-
- _Requirements: 7.1, 7.2, 7.3, 7.4_
|
| 72 |
-
|
| 73 |
-
- [ ] 10. Create configuration management and deployment setup
|
| 74 |
-
- Add Docker configuration for Guardrails server deployment
|
| 75 |
-
- Create environment-based configuration management
|
| 76 |
-
- Implement graceful fallback when Guardrails server is unavailable
|
| 77 |
-
- Add production deployment scripts and documentation
|
| 78 |
-
- Write end-to-end integration tests for complete system
|
| 79 |
-
- _Requirements: 5.1, 5.4, 1.4_
|
| 80 |
-
|
| 81 |
-
- [ ] 11. Update existing application integration points
|
| 82 |
-
- Modify existing node classes to use enhanced guardrails
|
| 83 |
-
- Update Streamlit UI to display new validation feedback
|
| 84 |
-
- Integrate structured data generation with existing workflows
|
| 85 |
-
- Add configuration options in UI for guard selection
|
| 86 |
-
- Write integration tests for updated application components
|
| 87 |
-
- _Requirements: 2.2, 3.2, 4.1, 6.2_
|
| 88 |
-
|
| 89 |
-
- [ ] 12. Implement comprehensive testing and validation
|
| 90 |
-
- Create performance benchmarks for validation latency
|
| 91 |
-
- Add security tests for malicious input handling
|
| 92 |
-
- Implement load testing for Guardrails server
|
| 93 |
-
- Create end-to-end tests for all validation scenarios
|
| 94 |
-
- Add documentation and examples for new features
|
| 95 |
-
- _Requirements: 2.1, 3.1, 5.2, 7.1_
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|