Update app.py

app.py

@@ -7,6 +7,8 @@ Website: https://thyris.ai
 
 import gradio as gr
 import re
+import uuid
+import hashlib
 from dataclasses import dataclass
 from typing import List, Tuple
 
@@ -14,77 +16,73 @@ from typing import List, Tuple
 class Detection:
     type: str
     value: str
+    placeholder: str
     start: int
     end: int
     confidence: str
+    mask_id: str
 
 class TSZDetector:
     PATTERNS = {
         "EMAIL": {
             "pattern": r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b',
-            "placeholder": "[EMAIL]",
             "description": "Email addresses"
         },
         "PHONE": {
             "pattern": r'\b(?:\+?1[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}\b',
-            "placeholder": "[PHONE]",
             "description": "Phone numbers (US format)"
         },
         "PHONE_TR": {
             "pattern": r'\b(?:\+90|0)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{3}[-.\s]?[0-9]{2}[-.\s]?[0-9]{2}\b',
-            "placeholder": "[PHONE]",
             "description": "Phone numbers (Turkey format)"
         },
         "CREDIT_CARD": {
             "pattern": r'\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[0-9]{12})\b',
-            "placeholder": "[CREDIT_CARD]",
             "description": "Credit card numbers"
         },
         "SSN": {
             "pattern": r'\b[0-9]{3}-[0-9]{2}-[0-9]{4}\b',
-            "placeholder": "[SSN]",
             "description": "Social Security Numbers (US)"
         },
         "TC_KIMLIK": {
             "pattern": r'\b[1-9][0-9]{10}\b',
-            "placeholder": "[TC_KIMLIK]",
             "description": "Turkish National ID"
         },
         "IBAN": {
             "pattern": r'\b[A-Z]{2}[0-9]{2}[A-Z0-9]{4}[0-9]{7}([A-Z0-9]?){0,16}\b',
-            "placeholder": "[IBAN]",
             "description": "International Bank Account Numbers"
         },
         "IP_ADDRESS": {
             "pattern": r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b',
-            "placeholder": "[IP_ADDRESS]",
             "description": "IPv4 addresses"
         },
         "API_KEY": {
             "pattern": r'\b(?:sk-[a-zA-Z0-9]{32,}|api[_-]?key[_-]?[=:]\s*["\']?[a-zA-Z0-9]{16,}["\']?)\b',
-            "placeholder": "[API_KEY]",
             "description": "API keys and secrets"
         },
         "AWS_KEY": {
             "pattern": r'\b(?:AKIA|ABIA|ACCA|ASIA)[A-Z0-9]{16}\b',
-            "placeholder": "[AWS_ACCESS_KEY]",
             "description": "AWS Access Keys"
         },
         "PASSWORD": {
             "pattern": r'(?:password|passwd|pwd)[_\s]*[=:]\s*["\']?[^\s"\']{4,}["\']?',
-            "placeholder": "[PASSWORD]",
             "description": "Passwords in config/code"
         },
         "JWT_TOKEN": {
             "pattern": r'\beyJ[A-Za-z0-9-_]+\.eyJ[A-Za-z0-9-_]+\.[A-Za-z0-9-_]+\b',
-            "placeholder": "[JWT_TOKEN]",
             "description": "JSON Web Tokens"
         }
     }
     
-    def detect(self, text: str) -> Tuple[List[Detection], str]:
+    def _generate_mask_id(self, value: str) -> str:
+        """Generate a short unique mask ID based on value hash"""
+        hash_val = hashlib.md5(value.encode()).hexdigest()[:6]
+        return hash_val
+    
+    def detect(self, text: str, rid: str = "NO-RID") -> Tuple[List[Detection], str]:
         detections = []
         redacted_text = text
+        type_counters = {}
         
         for pii_type, config in self.PATTERNS.items():
             pattern = config["pattern"]
@@ -92,32 +90,51 @@ class TSZDetector:
                 value = match.group()
                 if pii_type == "TC_KIMLIK" and len(value) != 11:
                     continue
+                
+                # Generate unique mask ID
+                if pii_type not in type_counters:
+                    type_counters[pii_type] = 0
+                type_counters[pii_type] += 1
+                
+                mask_id = self._generate_mask_id(value)
+                
+                # Format: [RID_TYPE_maskId]
+                placeholder = f"[{rid}_{pii_type}_{mask_id}]"
+                
                 detections.append(Detection(
                     type=pii_type,
                     value=value,
+                    placeholder=placeholder,
                     start=match.start(),
                     end=match.end(),
-                    confidence="HIGH" if len(value) > 8 else "MEDIUM"
+                    confidence="HIGH" if len(value) > 8 else "MEDIUM",
+                    mask_id=mask_id
                 ))
         
+        # Sort by position (reverse) for correct replacement
         detections.sort(key=lambda x: x.start, reverse=True)
         for det in detections:
-            placeholder = self.PATTERNS[det.type]["placeholder"]
-            redacted_text = redacted_text[:det.start] + placeholder + redacted_text[det.end:]
+            redacted_text = redacted_text[:det.start] + det.placeholder + redacted_text[det.end:]
+        
+        # Re-sort for display
         detections.sort(key=lambda x: x.start)
         
         return detections, redacted_text
 
 detector = TSZDetector()
 
-def analyze_text(text: str) -> Tuple[str, str, str]:
+def analyze_text(text: str, rid: str) -> Tuple[str, str, str, str]:
     if not text.strip():
-        return "", "Please enter some text to analyze.", ""
+        return "", "Please enter some text to analyze.", "", ""
+    
+    # Use provided RID or generate one
+    if not rid.strip():
+        rid = f"TSZ-{uuid.uuid4().hex[:8].upper()}"
     
-    detections, redacted_text = detector.detect(text)
+    detections, redacted_text = detector.detect(text, rid)
     
     if not detections:
-        return text, "No sensitive information detected.", "**0** PII entities found"
+        return text, "No sensitive information detected.", "**0** PII entities found", f"Request ID: `{rid}`"
     
     report_lines = ["### Detected Entities\n"]
     by_type = {}
@@ -131,18 +148,20 @@ def analyze_text(text: str) -> Tuple[str, str, str]:
         report_lines.append(f"\n**{pii_type}** ({desc})")
         for item in items:
             masked_value = item.value[:3] + "***" + item.value[-2:] if len(item.value) > 5 else "***"
-            report_lines.append(f"- `{masked_value}` (confidence: {item.confidence})")
+            report_lines.append(f"- `{masked_value}` → `{item.placeholder}`")
+            report_lines.append(f"  - Mask ID: `{item.mask_id}` | Confidence: {item.confidence}")
     
     report = "\n".join(report_lines)
     stats = f"**{len(detections)}** PII entities found across **{len(by_type)}** categories"
+    rid_display = f"Request ID: `{rid}`"
     
-    return redacted_text, report, stats
+    return redacted_text, report, stats, rid_display
 
 EXAMPLES = [
-    ["Hi, my name is John Smith and my email is john.smith@company.com. You can reach me at +1-555-123-4567. My SSN is 123-45-6789."],
-    ["Customer Order #12345\nName: Jane Doe\nEmail: jane.doe@example.org\nPhone: (555) 987-6543\nCredit Card: 4532015112830366"],
-    ["API Configuration:\napi_key = \"sk-1234567890abcdefghijklmnopqrstuvwxyz\"\nAWS_ACCESS_KEY: AKIAIOSFODNN7EXAMPLE\npassword: \"super_secret_123\""],
-    ["Merhaba, ben Ahmet Yilmaz. TC Kimlik numaram 12345678901.\nTelefon: 0532 123 45 67\nIBAN: TR330006100519786457841326"],
+    ["Hi, my name is John Smith and my email is john.smith@company.com. You can reach me at +1-555-123-4567. My SSN is 123-45-6789.", "REQ-001"],
+    ["Customer Order #12345\nName: Jane Doe\nEmail: jane.doe@example.org\nPhone: (555) 987-6543\nCredit Card: 4532015112830366", "ORDER-12345"],
+    ["API Configuration:\napi_key = \"sk-1234567890abcdefghijklmnopqrstuvwxyz\"\nAWS_ACCESS_KEY: AKIAIOSFODNN7EXAMPLE\npassword: \"super_secret_123\"", "CONFIG-SCAN"],
+    ["Merhaba, ben Ahmet Yilmaz. TC Kimlik numaram 12345678901.\nTelefon: 0532 123 45 67\nIBAN: TR330006100519786457841326", "TR-REQ-001"],
 ]
 
 with gr.Blocks(title="TSZ - Thyris Safe Zone Demo", theme=gr.themes.Soft()) as demo:
@@ -161,6 +180,11 @@ with gr.Blocks(title="TSZ - Thyris Safe Zone Demo", theme=gr.themes.Soft()) as d
         with gr.TabItem("🔍 Interactive Demo"):
             with gr.Row():
                 with gr.Column():
+                    rid_input = gr.Textbox(
+                        label="Request ID (RID)",
+                        placeholder="e.g., REQ-001 (auto-generated if empty)",
+                        lines=1
+                    )
                     input_text = gr.Textbox(
                         label="Input Text",
                         placeholder="Paste text containing emails, phone numbers, credit cards, API keys...",
@@ -168,9 +192,10 @@ with gr.Blocks(title="TSZ - Thyris Safe Zone Demo", theme=gr.themes.Soft()) as d
                     )
                     analyze_btn = gr.Button("Analyze & Redact", variant="primary")
                     gr.Markdown("### Try These Examples")
-                    gr.Examples(examples=EXAMPLES, inputs=input_text)
+                    gr.Examples(examples=EXAMPLES, inputs=[input_text, rid_input])
                 
                 with gr.Column():
+                    rid_display = gr.Markdown(label="Request ID")
                     stats_output = gr.Markdown(label="Statistics")
                     redacted_output = gr.Textbox(
                         label="Redacted Output (Safe to send to LLMs)",
@@ -179,38 +204,37 @@ with gr.Blocks(title="TSZ - Thyris Safe Zone Demo", theme=gr.themes.Soft()) as d
                     )
                     detection_report = gr.Markdown(label="Detection Report")
             
-            analyze_btn.click(fn=analyze_text, inputs=[input_text], outputs=[redacted_output, detection_report, stats_output])
-            input_text.submit(fn=analyze_text, inputs=[input_text], outputs=[redacted_output, detection_report, stats_output])
+            gr.Markdown("""
+            ---
+            ### Placeholder Format
+            
+            TSZ uses the format `[RID_TYPE_maskId]` for placeholders:
+            - **RID**: Request ID for audit correlation
+            - **TYPE**: Detection type (EMAIL, PHONE, etc.)
+            - **maskId**: Unique identifier derived from the original value
+            
+            Example: `[REQ-001_EMAIL_a1b2c3]`
+            
+            This allows you to:
+            - Track which request generated the redaction
+            - Identify the type of sensitive data
+            - Correlate with audit logs for compliance
+            """)
+            
+            analyze_btn.click(fn=analyze_text, inputs=[input_text, rid_input], outputs=[redacted_output, detection_report, stats_output, rid_display])
+            input_text.submit(fn=analyze_text, inputs=[input_text, rid_input], outputs=[redacted_output, detection_report, stats_output, rid_display])
         
         # Tab 2: API Reference
         with gr.TabItem("📚 API Reference"):
             gr.Markdown("""
 ## TSZ API Reference
 
-TSZ is an enterprise-grade PII detection and guardrails gateway. It acts as a zero-trust middleware between your applications and external systems (LLMs, SaaS APIs, third-party services).
-
----
-
-### Base Information
-
-**Base URL (Docker Compose):**
-```
-http://localhost:8080
-```
-
-**Production:**
-```
-https://tsz.your-company.com
-```
-
-**Content Type:** `application/json`
+TSZ is an enterprise-grade PII detection and guardrails gateway. It acts as a zero-trust middleware between your applications and external systems.
 
 ---
 
 ### Confidence & Guardrails Model
 
-TSZ uses a hybrid confidence system:
-
 | Confidence | Action |
 |------------|--------|
 | < 0.30 | ALLOW (ignored) |
@@ -221,13 +245,11 @@ TSZ uses a hybrid confidence system:
 
 ### POST /detect
 
-Primary endpoint for PII detection and redaction.
-
 **Request:**
 ```json
 {
   "text": "My email is user@company.com",
-  "rid": "request-123",
+  "rid": "REQ-001",
   "guardrails": ["TOXIC_LANGUAGE"]
 }
 ```
@@ -235,12 +257,12 @@ Primary endpoint for PII detection and redaction.
 **Response:**
 ```json
 {
-  "redacted_text": "My email is [EMAIL]",
+  "redacted_text": "My email is [REQ-001_EMAIL_a1b2c3]",
   "detections": [
     {
       "type": "EMAIL",
       "value": "user@company.com",
-      "placeholder": "[EMAIL]",
+      "placeholder": "[REQ-001_EMAIL_a1b2c3]",
       "start": 12,
       "end": 28,
       "confidence_score": "0.78",
@@ -258,17 +280,22 @@ Primary endpoint for PII detection and redaction.
 }
 ```
 
-**cURL Example:**
-```bash
-curl -X POST http://localhost:8080/detect \\
-  -H "Content-Type: application/json" \\
-  -d '{
-    "text": "Contact john@example.com",
-    "rid": "req-001",
-    "guardrails": ["TOXIC_LANGUAGE"]
-  }'
+---
+
+### Placeholder Format
+
+TSZ generates unique placeholders in the format:
+
+```
+[{RID}_{TYPE}_{maskId}]
 ```
 
+- **RID**: Request ID for audit log correlation
+- **TYPE**: Detection type (EMAIL, CREDIT_CARD, etc.)
+- **maskId**: Hash-based unique identifier
+
+Example: `[RID-GW-001_EMAIL_a1b2c3]`
+
 ---
 
 ### POST /v1/chat/completions
@@ -277,41 +304,27 @@ OpenAI-compatible LLM Gateway with built-in guardrails.
 
 **Headers:**
 - `X-TSZ-RID`: Request ID for audit logs
-- `X-TSZ-Guardrails`: Comma-separated validators (e.g., `TOXIC_LANGUAGE,PII`)
+- `X-TSZ-Guardrails`: Comma-separated validators
 - `X-TSZ-Guardrails-Mode`: `final-only` | `stream-sync` | `stream-async`
-- `X-TSZ-Guardrails-OnFail`: `filter` | `halt`
 
-**Request:**
-```bash
-curl -X POST http://localhost:8080/v1/chat/completions \\
-  -H "Content-Type: application/json" \\
-  -H "X-TSZ-Guardrails: TOXIC_LANGUAGE" \\
-  -d '{
-    "model": "gpt-4",
-    "messages": [
-      {"role": "user", "content": "My credit card is 4111111111111111"}
+**Response with tsz_meta:**
+```json
+{
+  "id": "chatcmpl-58",
+  "choices": [...],
+  "tsz_meta": {
+    "rid": "RID-GW-001",
+    "guardrails": ["TOXIC_LANGUAGE"],
+    "input": [
+      {
+        "redacted_text": "My email is [RID-GW-001_EMAIL_a1b2c3]",
+        "detections": [...],
+        "blocked": false
+      }
     ],
-    "stream": false
-  }'
-```
-
-**Python SDK Example:**
-```python
-from openai import OpenAI
-
-client = OpenAI(
-    base_url="http://localhost:8080/v1",
-    api_key="dummy"  # TSZ uses env var
-)
-
-resp = client.chat.completions.create(
-    model="gpt-4",
-    messages=[{"role": "user", "content": "Hello"}],
-    extra_headers={
-        "X-TSZ-Guardrails": "TOXIC_LANGUAGE,PII",
-        "X-TSZ-Guardrails-Mode": "stream-sync"
-    }
-)
+    "output": [...]
+  }
+}
 ```
 
 ---
@@ -324,41 +337,10 @@ resp = client.chat.completions.create(
 | `GET` | `/patterns` | List patterns |
 | `DELETE` | `/patterns/{id}` | Delete pattern |
 
-**Create Pattern:**
-```json
-{
-  "Name": "PHONE_NUMBER",
-  "Regex": "\\\\+?[0-9]{10,13}",
-  "Category": "PII",
-  "IsActive": true,
-  "BlockThreshold": 0.9
-}
-```
-
----
-
-### Allowlist / Blocklist
-
-| Method | Endpoint | Description |
-|--------|----------|-------------|
-| `POST` | `/allowlist` | Add trusted value |
-| `GET` | `/allowlist` | List allowlist |
-| `DELETE` | `/allowlist/{id}` | Remove item |
-| `POST` | `/blacklist` | Add blocked value |
-| `GET` | `/blacklist` | List blocklist |
-| `DELETE` | `/blacklist/{id}` | Remove item |
-
 ---
 
 ### Validators & Guardrails
 
-| Method | Endpoint | Description |
-|--------|----------|-------------|
-| `POST` | `/validators` | Create validator |
-| `GET` | `/validators` | List validators |
-| `DELETE` | `/validators/{id}` | Delete validator |
-
-**AI Validator Example:**
 ```json
 {
   "name": "TOXIC_LANGUAGE",
@@ -375,33 +357,11 @@ resp = client.chat.completions.create(
 | Endpoint | Description |
 |----------|-------------|
 | `GET /healthz` | Liveness probe |
-| `GET /ready` | Readiness probe (DB + Redis) |
+| `GET /ready` | Readiness probe |
 | `POST /admin/reload` | Clear caches |
 
 ---
 
-### Client SDKs
-
-**Python:**
-```bash
-pip install "tszclient-py @ git+https://github.com/thyrisAI/safe-zone.git@main"
-```
-
-```python
-from tszclient_py import TSZClient
-client = TSZClient("http://localhost:8080")
-result = client.detect("test@example.com")
-```
-
-**Go:**
-```go
-import "github.com/thyrisAI/safe-zone/pkg/tszclient-go"
-client := tszclient.New("http://localhost:8080")
-result, _ := client.Detect("test@example.com")
-```
-
----
-
 📖 **Full Documentation:** [API_REFERENCE.md](https://github.com/thyrisAI/safe-zone/blob/main/docs/API_REFERENCE.md)
             """)
         
@@ -428,24 +388,36 @@ TSZ will be available at `http://localhost:8080`
 # LLM Gateway Configuration
 THYRIS_AI_MODEL_URL=https://api.openai.com/v1
 THYRIS_AI_API_KEY=sk-your-key
-THYRIS_AI_MODEL=gpt-4
 
 # Confidence Thresholds
 CONFIDENCE_ALLOW_THRESHOLD=0.30
 CONFIDENCE_BLOCK_THRESHOLD=0.85
 
-# Admin API Key
-ADMIN_API_KEY=your-secure-admin-key
+# PII Mode
+PII_MODE=MASK  # or BLOCK
+GATEWAY_BLOCK_MODE=BLOCK  # BLOCK, MASK, or WARN
 ```
 
 ---
 
-### Test the API
+### Test with Request ID
 
 ```bash
 curl -X POST http://localhost:8080/detect \\
   -H "Content-Type: application/json" \\
-  -d '{"text": "Email: test@example.com", "rid": "test-001"}'
+  -d '{
+    "text": "Email: test@example.com",
+    "rid": "MY-REQ-001"
+  }'
+```
+
+**Response:**
+```json
+{
+  "redacted_text": "Email: [MY-REQ-001_EMAIL_abc123]",
+  "detections": [...],
+  "contains_pii": true
+}
 ```
 
 ---
@@ -456,28 +428,15 @@ curl -X POST http://localhost:8080/detect \\
 ┌─────────────┐     ┌─────────────┐     ┌─────────────┐
 │  Your App   │────▶│     TSZ     │────▶│  LLM API    │
 └─────────────┘     └─────────────┘     └─────────────┘
-                          │
-                    ┌─────▼─────┐
-                    │ Audit Log │
-                    │  + Redis  │
-                    └───────────┘
+       │                  │
+       │            ┌─────▼─────┐
+       │            │ Audit Log │
+       │            │ (with RID)│
+       └───────────▶└───────────┘
 ```
 
 ---
 
-### Key Features
-
-| Feature | Description |
-|---------|-------------|
-| **Hybrid Detection** | Regex + AI-powered confidence scoring |
-| **Real-time Redaction** | Context-preserving placeholders |
-| **LLM Gateway** | OpenAI-compatible proxy with guardrails |
-| **Streaming Support** | sync/async modes for SSE |
-| **Hot-reload Rules** | Update patterns via API |
-| **Audit Trail** | Full logging for compliance |
-
----
-
 📖 **Documentation:** [github.com/thyrisAI/safe-zone/docs](https://github.com/thyrisAI/safe-zone/tree/main/docs)
             """)
         
@@ -488,26 +447,35 @@ curl -X POST http://localhost:8080/detect \\
 
 Developed by **Thyris.AI** as an open-source enterprise AI security solution.
 
-### Why TSZ?
+### Key Features
 
-| Challenge | TSZ Solution |
-|-----------|--------------|
-| PII leaking to LLMs | Real-time detection & redaction |
-| GDPR/KVKK compliance | Data never leaves your perimeter |
-| Toxic content in AI | AI-powered guardrails |
-| Audit requirements | Complete request logging |
-| Integration complexity | OpenAI-compatible gateway |
+| Feature | Description |
+|---------|-------------|
+| **Hybrid Detection** | Regex + AI-powered confidence scoring |
+| **Traceable Redaction** | `[RID_TYPE_maskId]` format for audit |
+| **LLM Gateway** | OpenAI-compatible proxy |
+| **Streaming Support** | sync/async modes for SSE |
+| **Guardrails** | AI validators (toxic, schema, custom) |
 
 ---
 
-### Detection Types
+### Placeholder System
 
-| Category | Types |
-|----------|-------|
-| **Personal Info** | Email, Phone, SSN, TC Kimlik, DOB |
-| **Financial** | Credit Card, IBAN |
-| **Technical** | API Keys, AWS Keys, Passwords, JWT |
-| **Custom** | User-defined patterns |
+Each redacted value gets a unique, traceable placeholder:
+
+```
+[RID-GW-001_EMAIL_a1b2c3]
+  │          │      │
+  │          │      └── Unique mask ID (hash of value)
+  │          └── Detection type
+  └── Request ID for correlation
+```
+
+Benefits:
+- Full audit trail
+- SIEM integration
+- Compliance reporting
+- Debug & investigation
 
 ---
 
@@ -521,15 +489,6 @@ Developed by **Thyris.AI** as an open-source enterprise AI security solution.
 
 ---
 
-### Links
-
-- 🔗 **GitHub:** [thyrisAI/safe-zone](https://github.com/thyrisAI/safe-zone)
-- 🌐 **Website:** [thyris.ai](https://thyris.ai)
-- 📄 **License:** Apache 2.0
-- 📖 **Docs:** [Documentation](https://github.com/thyrisAI/safe-zone/tree/main/docs)
-
----
-
 Built with ❤️ by [Thyris.AI](https://thyris.ai)
             """)