Sync: Can now hide duplicate pages or Excel/Word file redaction tabs

.coveragerc

@@ -0,0 +1,56 @@
+[run]
+source = .
+omit = 
+    */tests/*
+    */test/*
+    */__pycache__/*
+    */venv/*
+    */env/*
+    */build/*
+    */dist/*
+    */cdk/*
+    */docs/*
+    */example_data/*
+    */examples/*
+    */feedback/*
+    */logs/*
+    */old_code/*
+    */output/*
+    */tmp/*
+    */usage/*
+    */tld/*
+    */tesseract/*
+    */poppler/*
+    config*.py
+    setup.py
+    lambda_entrypoint.py
+    entrypoint.sh
+    cli_redact.py
+    load_dynamo_logs.py
+    load_s3_logs.py
+    *.spec
+    Dockerfile
+    *.qmd
+    *.md
+    *.txt
+    *.yml
+    *.yaml
+    *.json
+    *.csv
+    *.env
+    *.bat
+    *.ps1
+    *.sh
+
+[report]
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    if self.debug:
+    if settings.DEBUG
+    raise AssertionError
+    raise NotImplementedError
+    if 0:
+    if __name__ == .__main__.:
+    class .*\bProtocol\):
+    @(abc\.)?abstractmethod

.dockerignore

@@ -0,0 +1,58 @@
+*.url
+*.ipynb
+*.pyc
+*.qmd
+*.json.bak.*
+_quarto.yml
+quarto_site/*
+src/*
+redaction_deps/*
+.venv/*
+examples/*
+processing/*
+tools/__pycache__/*
+old_code/*
+tesseract/*
+poppler/*
+build/*
+dist/*
+docs/*
+.pi/*
+build_deps/*
+user_guide/*
+_extensions/*
+workspace/*
+doc_redaction.egg-info/*
+.venv_pypi_test/*
+cdk/config/*
+tld/*
+cdk/config/*
+cdk/cdk.out/*
+cdk/archive/*
+cdk.json
+cdk.context.json
+.quarto/*
+logs/
+output/
+input/
+feedback/
+# Exclude local secrets; allow committed *.example templates (Pi agent + main app images).
+config/*
+!config/agent.env.example
+!config/app_config.env.example
+!config/docker_app_config.env.example
+usage/
+test/config/*
+test/feedback/*
+test/input/*
+test/logs/*
+test/output/*
+test/tmp/*
+test/usage/*
+.ruff_cache/*
+model_cache/*
+sanitized_file/*
+src/doc_redaction.egg-info/*
+docker_compose/*
+skills/example_prompts/*
+agent-redact/RedactionAgent/*

.gitattributes

@@ -0,0 +1,9 @@
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.sh text eol=lf
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.xls filter=lfs diff=lfs merge=lfs -text
+*.xlsx filter=lfs diff=lfs merge=lfs -text
+*.docx filter=lfs diff=lfs merge=lfs -text
+*.doc filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.ico filter=lfs diff=lfs merge=lfs -text

.github/scripts/setup_test_data.py

@@ -0,0 +1,320 @@
+#!/usr/bin/env python3
+"""
+Setup script for GitHub Actions test data.
+Creates dummy test files when example data is not available.
+"""
+
+import os
+import sys
+
+import pandas as pd
+
+
+def create_directories():
+    """Create necessary directories."""
+    dirs = ["doc_redaction/example_data", "doc_redaction/example_data/example_outputs"]
+
+    for dir_path in dirs:
+        os.makedirs(dir_path, exist_ok=True)
+        print(f"Created directory: {dir_path}")
+
+
+def create_dummy_pdf():
+    """Create dummy PDFs for testing."""
+
+    # Install reportlab if not available
+    try:
+        from reportlab.lib.pagesizes import letter
+        from reportlab.pdfgen import canvas
+    except ImportError:
+        import subprocess
+
+        subprocess.check_call(["pip", "install", "reportlab"])
+        from reportlab.lib.pagesizes import letter
+        from reportlab.pdfgen import canvas
+
+    try:
+        # Create the main test PDF
+        pdf_path = "doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf"
+        print(f"Creating PDF: {pdf_path}")
+        print(f"Directory exists: {os.path.exists('doc_redaction/example_data')}")
+
+        c = canvas.Canvas(pdf_path, pagesize=letter)
+        c.drawString(100, 750, "This is a test document for redaction testing.")
+        c.drawString(100, 700, "Email: test@example.com")
+        c.drawString(100, 650, "Phone: 123-456-7890")
+        c.drawString(100, 600, "Name: John Doe")
+        c.drawString(100, 550, "Address: 123 Test Street, Test City, TC 12345")
+        c.showPage()
+
+        # Add second page
+        c.drawString(100, 750, "Second page content")
+        c.drawString(100, 700, "More test data: jane.doe@example.com")
+        c.drawString(100, 650, "Another phone: 987-654-3210")
+        c.save()
+
+        print(f"Created dummy PDF: {pdf_path}")
+
+        # Create Partnership Agreement Toolkit PDF
+        partnership_pdf_path = (
+            "doc_redaction/example_data/Partnership-Agreement-Toolkit_0_0.pdf"
+        )
+        print(f"Creating PDF: {partnership_pdf_path}")
+        c = canvas.Canvas(partnership_pdf_path, pagesize=letter)
+        c.drawString(100, 750, "Partnership Agreement Toolkit")
+        c.drawString(100, 700, "This is a test partnership agreement document.")
+        c.drawString(100, 650, "Contact: partnership@example.com")
+        c.drawString(100, 600, "Phone: (555) 123-4567")
+        c.drawString(100, 550, "Address: 123 Partnership Street, City, State 12345")
+        c.showPage()
+
+        # Add second page
+        c.drawString(100, 750, "Page 2 - Partnership Details")
+        c.drawString(100, 700, "More partnership information here.")
+        c.drawString(100, 650, "Contact: info@partnership.org")
+        c.showPage()
+
+        # Add third page
+        c.drawString(100, 750, "Page 3 - Terms and Conditions")
+        c.drawString(100, 700, "Terms and conditions content.")
+        c.drawString(100, 650, "Legal contact: legal@partnership.org")
+        c.save()
+
+        print(f"Created dummy PDF: {partnership_pdf_path}")
+
+        # Create Graduate Job Cover Letter PDF
+        cover_letter_pdf_path = (
+            "doc_redaction/example_data/graduate-job-example-cover-letter.pdf"
+        )
+        print(f"Creating PDF: {cover_letter_pdf_path}")
+        c = canvas.Canvas(cover_letter_pdf_path, pagesize=letter)
+        c.drawString(100, 750, "Cover Letter Example")
+        c.drawString(100, 700, "Dear Hiring Manager,")
+        c.drawString(100, 650, "I am writing to apply for the position.")
+        c.drawString(100, 600, "Contact: applicant@example.com")
+        c.drawString(100, 550, "Phone: (555) 987-6543")
+        c.drawString(100, 500, "Address: 456 Job Street, Employment City, EC 54321")
+        c.drawString(100, 450, "Sincerely,")
+        c.drawString(100, 400, "John Applicant")
+        c.save()
+
+        print(f"Created dummy PDF: {cover_letter_pdf_path}")
+
+    except ImportError:
+        print("ReportLab not available, skipping PDF creation")
+        # Create simple text files instead
+        with open(
+            "doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf",
+            "w",
+        ) as f:
+            f.write("This is a dummy PDF file for testing")
+
+        with open(
+            "doc_redaction/example_data/Partnership-Agreement-Toolkit_0_0.pdf",
+            "w",
+        ) as f:
+            f.write("This is a dummy Partnership Agreement PDF file for testing")
+
+        with open(
+            "doc_redaction/example_data/graduate-job-example-cover-letter.pdf",
+            "w",
+        ) as f:
+            f.write("This is a dummy cover letter PDF file for testing")
+
+        print("Created dummy text files instead of PDFs")
+
+
+def create_dummy_csv():
+    """Create dummy CSV files for testing."""
+    # Main CSV
+    csv_data = {
+        "Case Note": [
+            "Client visited for consultation regarding housing issues",
+            "Follow-up appointment scheduled for next week",
+            "Documentation submitted for review",
+        ],
+        "Client": ["John Smith", "Jane Doe", "Bob Johnson"],
+        "Date": ["2024-01-15", "2024-01-16", "2024-01-17"],
+    }
+    df = pd.DataFrame(csv_data)
+    df.to_csv("doc_redaction/example_data/combined_case_notes.csv", index=False)
+    print("Created dummy CSV: doc_redaction/example_data/combined_case_notes.csv")
+
+    # Lambeth CSV
+    lambeth_data = {
+        "text": [
+            "Lambeth 2030 vision document content",
+            "Our Future Our Lambeth strategic plan",
+            "Community engagement and development",
+        ],
+        "page": [1, 2, 3],
+    }
+    df_lambeth = pd.DataFrame(lambeth_data)
+    df_lambeth.to_csv(
+        "doc_redaction/example_data/Lambeth_2030-Our_Future_Our_Lambeth.pdf.csv",
+        index=False,
+    )
+    print(
+        "Created dummy CSV: doc_redaction/example_data/Lambeth_2030-Our_Future_Our_Lambeth.pdf.csv"
+    )
+
+
+def create_dummy_word_doc():
+    """Create dummy Word document."""
+    try:
+        from docx import Document
+
+        doc = Document()
+        doc.add_heading("Test Document for Redaction", 0)
+        doc.add_paragraph("This is a test document for redaction testing.")
+        doc.add_paragraph("Contact Information:")
+        doc.add_paragraph("Email: test@example.com")
+        doc.add_paragraph("Phone: 123-456-7890")
+        doc.add_paragraph("Name: John Doe")
+        doc.add_paragraph("Address: 123 Test Street, Test City, TC 12345")
+
+        doc.save(
+            "doc_redaction/example_data/Bold minimalist professional cover letter.docx"
+        )
+        print("Created dummy Word document")
+
+    except ImportError:
+        print("python-docx not available, skipping Word document creation")
+
+
+def create_allow_deny_lists():
+    """Create dummy allow/deny lists."""
+    # Allow lists
+    allow_data = {"word": ["test", "example", "document"]}
+    pd.DataFrame(allow_data).to_csv(
+        "doc_redaction/example_data/test_allow_list_graduate.csv", index=False
+    )
+    pd.DataFrame(allow_data).to_csv(
+        "doc_redaction/example_data/test_allow_list_partnership.csv", index=False
+    )
+    print("Created allow lists")
+
+    # Deny lists
+    deny_data = {"word": ["sensitive", "confidential", "private"]}
+    pd.DataFrame(deny_data).to_csv(
+        "doc_redaction/example_data/partnership_toolkit_redact_custom_deny_list.csv",
+        index=False,
+    )
+    pd.DataFrame(deny_data).to_csv(
+        "doc_redaction/example_data/Partnership-Agreement-Toolkit_test_deny_list_para_single_spell.csv",
+        index=False,
+    )
+    print("Created deny lists")
+
+    # Whole page redaction list
+    page_data = {"page": [1, 2]}
+    pd.DataFrame(page_data).to_csv(
+        "doc_redaction/example_data/partnership_toolkit_redact_some_pages.csv",
+        index=False,
+    )
+    print("Created whole page redaction list")
+
+
+def create_ocr_output():
+    """Create dummy OCR output CSV."""
+    ocr_data = {
+        "page": [1, 2, 3],
+        "text": [
+            "This is page 1 content with some text",
+            "This is page 2 content with different text",
+            "This is page 3 content with more text",
+        ],
+        "left": [0.1, 0.3, 0.5],
+        "top": [0.95, 0.92, 0.88],
+        "width": [0.05, 0.02, 0.02],
+        "height": [0.01, 0.02, 0.02],
+        "line": [1, 2, 3],
+    }
+    df = pd.DataFrame(ocr_data)
+    df.to_csv(
+        "doc_redaction/example_data/example_outputs/doubled_output_joined.pdf_ocr_output.csv",
+        index=False,
+    )
+    print("Created dummy OCR output CSV")
+
+
+def create_dummy_image():
+    """Create dummy image for testing."""
+    try:
+        from PIL import Image, ImageDraw, ImageFont
+
+        img = Image.new("RGB", (800, 600), color="white")
+        draw = ImageDraw.Draw(img)
+
+        # Try to use a system font
+        try:
+            font = ImageFont.truetype(
+                "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 20
+            )
+        except Exception as e:
+            print(f"Error loading DejaVuSans font: {e}")
+            try:
+                font = ImageFont.truetype("/System/Library/Fonts/Arial.ttf", 20)
+            except Exception as e:
+                print(f"Error loading Arial font: {e}")
+                font = ImageFont.load_default()
+
+        # Add text to image
+        draw.text((50, 50), "Test Document for Redaction", fill="black", font=font)
+        draw.text((50, 100), "Email: test@example.com", fill="black", font=font)
+        draw.text((50, 150), "Phone: 123-456-7890", fill="black", font=font)
+        draw.text((50, 200), "Name: John Doe", fill="black", font=font)
+        draw.text((50, 250), "Address: 123 Test Street", fill="black", font=font)
+
+        img.save("doc_redaction/example_data/example_complaint_letter.jpg")
+        print("Created dummy image")
+
+    except ImportError:
+        print("PIL not available, skipping image creation")
+
+
+def main():
+    """Main setup function."""
+    print("Setting up test data for GitHub Actions...")
+    print(f"Current working directory: {os.getcwd()}")
+    print(f"Python version: {sys.version}")
+
+    create_directories()
+    create_dummy_pdf()
+    create_dummy_csv()
+    create_dummy_word_doc()
+    create_allow_deny_lists()
+    create_ocr_output()
+    create_dummy_image()
+
+    print("\nTest data setup complete!")
+    print("Created files:")
+    for root, dirs, files in os.walk("doc_redaction/example_data"):
+        for file in files:
+            file_path = os.path.join(root, file)
+            print(f"  {file_path}")
+            # Verify the file exists and has content
+            if os.path.exists(file_path):
+                file_size = os.path.getsize(file_path)
+                print(f"    Size: {file_size} bytes")
+            else:
+                print("    WARNING: File does not exist!")
+
+    # Verify critical files exist
+    critical_files = [
+        "doc_redaction/example_data/Partnership-Agreement-Toolkit_0_0.pdf",
+        "doc_redaction/example_data/graduate-job-example-cover-letter.pdf",
+        "doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf",
+    ]
+
+    print("\nVerifying critical test files:")
+    for file_path in critical_files:
+        if os.path.exists(file_path):
+            file_size = os.path.getsize(file_path)
+            print(f"✅ {file_path} exists ({file_size} bytes)")
+        else:
+            print(f"❌ {file_path} MISSING!")
+
+
+if __name__ == "__main__":
+    main()

.github/workflow_README.md

@@ -0,0 +1,183 @@
+# GitHub Actions CI/CD Setup
+
+This directory contains GitHub Actions workflows for automated testing of the CLI redaction application.
+
+## Workflows Overview
+
+### 1. **Simple Test Run** (`.github/workflows/simple-test.yml`)
+- **Purpose**: Basic test execution
+- **Triggers**: Push to main/dev, Pull requests
+- **OS**: Ubuntu Latest
+- **Python**: 3.11
+- **Features**: 
+  - Installs system dependencies
+  - Sets up test data
+  - Runs CLI tests
+  - Runs pytest
+
+### 2. **Comprehensive CI/CD** (`.github/workflows/ci.yml`)
+- **Purpose**: Full CI/CD pipeline
+- **Features**:
+  - Linting (Ruff, Black)
+  - Unit tests (Python 3.10, 3.11, 3.12)
+  - Integration tests
+  - Security scanning (Safety, Bandit)
+  - Coverage reporting
+  - Package building (on main branch)
+
+### 3. **Multi-OS Testing** (`.github/workflows/multi-os-test.yml`)
+- **Purpose**: Cross-platform testing
+- **OS**: Ubuntu, macOS (Windows not included currently but may be reintroduced)
+- **Python**: 3.10, 3.11, 3.12
+- **Features**: Tests compatibility across different operating systems
+
+### 4. **Basic Test Suite** (`.github/workflows/test.yml`)
+- **Purpose**: Original test workflow
+- **Features**: 
+  - Multiple Python versions
+  - System dependency installation
+  - Test data creation
+  - Coverage reporting
+
+## Setup Scripts
+
+### Test Data Setup (`.github/scripts/setup_test_data.py`)
+Creates dummy test files when example data is not available:
+- PDF documents
+- CSV files
+- Word documents
+- Images
+- Allow/deny lists
+- OCR output files
+
+## Usage
+
+### Running Tests Locally
+
+```bash
+# Install dependencies
+pip install -r requirements.txt
+pip install pytest pytest-cov
+
+# Setup test data
+python .github/scripts/setup_test_data.py
+
+# Run tests
+cd test
+python cli_epilog_suite.py
+```
+
+### GitHub Actions Triggers
+
+1. **Push to main/dev**: Runs all tests
+2. **Pull Request**: Runs tests and linting
+3. **Daily Schedule**: Runs tests at 2 AM UTC
+4. **Manual Trigger**: Can be triggered manually from GitHub
+
+## Configuration
+
+### Environment Variables
+- `PYTHON_VERSION`: Default Python version (3.11)
+- `PYTHONPATH`: Set automatically for test discovery
+
+### Caching
+- Pip dependencies are cached for faster builds
+- Cache key based on requirements.txt hash
+
+### Artifacts
+- Test results (JUnit XML)
+- Coverage reports (HTML, XML)
+- Security reports
+- Build artifacts (on main branch)
+
+## Test Data
+
+The workflows automatically create test data when example files are missing:
+
+### Required Files Created:
+- `example_data/example_of_emails_sent_to_a_professor_before_applying.pdf`
+- `example_data/combined_case_notes.csv`
+- `example_data/Bold minimalist professional cover letter.docx`
+- `example_data/example_complaint_letter.jpg`
+- `example_data/test_allow_list_*.csv`
+- `example_data/partnership_toolkit_redact_*.csv`
+- `example_data/example_outputs/doubled_output_joined.pdf_ocr_output.csv`
+
+### Dependencies Installed:
+- **System**: tesseract-ocr, poppler-utils, OpenGL libraries
+- **Python**: All requirements.txt packages + pytest, reportlab, pillow
+
+## Workflow Status
+
+### Success Criteria:
+- ✅ All tests pass
+- ✅ No linting errors
+- ✅ Security checks pass
+- ✅ Coverage meets threshold (if configured)
+
+### Failure Handling:
+- Tests are designed to skip gracefully if files are missing
+- AWS tests are expected to fail without credentials
+- System dependency failures are handled with fallbacks
+
+## Customization
+
+### Adding New Tests:
+1. Add test methods to `test/cli_epilog_suite.py` or pytest files under `test/test_*.py`
+2. Update test data in `setup_test_data.py` if needed
+3. Tests will automatically run in all workflows
+
+### Modifying Workflows:
+1. Edit the appropriate `.yml` file
+2. Test locally first
+3. Push to trigger the workflow
+
+### Environment-Specific Settings:
+- **Ubuntu**: Full system dependencies
+- **Windows**: Python packages only
+- **macOS**: Homebrew dependencies
+
+## Troubleshooting
+
+### Common Issues:
+
+1. **Missing Dependencies**:
+   - Check system dependency installation
+   - Verify Python package versions
+
+2. **Test Failures**:
+   - Check test data creation
+   - Verify file paths
+   - Review test output logs
+
+3. **AWS Test Failures**:
+   - Expected without credentials
+   - Tests are designed to handle this gracefully
+
+4. **System Dependency Issues**:
+   - Different OS have different requirements
+   - Check the specific OS section in workflows
+
+### Debug Mode:
+Add `--verbose` or `-v` flags to pytest commands for more detailed output.
+
+## Security
+
+- Dependencies are scanned with Safety
+- Code is scanned with Bandit
+- No secrets are exposed in logs
+- Test data is temporary and cleaned up
+
+## Performance
+
+- Tests run in parallel where possible
+- Dependencies are cached
+- Only necessary system packages are installed
+- Test data is created efficiently
+
+## Monitoring
+
+- Workflow status is visible in GitHub Actions tab
+- Coverage reports are uploaded to Codecov
+- Test results are available as artifacts
+- Security reports are generated and stored

.github/workflows/archive_workflows/multi-os-test.yml

@@ -0,0 +1,115 @@
+name: Multi-OS Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    env:
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest] # windows-latest, not included as tesseract cannot be installed silently
+        python-version: ["3.11", "3.12", "3.13"]
+        exclude:
+          # Exclude some combinations to reduce CI time
+          #- os: windows-latest
+          #  python-version: ["3.12", "3.13"]
+          - os: macos-latest
+            python-version: ["3.12", "3.13"]
+
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Install system dependencies (Ubuntu)
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Install system dependencies (macOS)
+      if: matrix.os == 'macos-latest'
+      run: |
+        brew install tesseract poppler
+    
+    - name: Install system dependencies (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        # Create tools directory
+        if (!(Test-Path "C:\tools")) {
+            mkdir C:\tools
+        }
+        
+        # Download and install Tesseract
+        $tesseractUrl = "https://github.com/tesseract-ocr/tesseract/releases/download/5.5.0/tesseract-ocr-w64-setup-5.5.0.20241111.exe"
+        $tesseractInstaller = "C:\tools\tesseract-installer.exe"
+        Invoke-WebRequest -Uri $tesseractUrl -OutFile $tesseractInstaller
+        
+        # Install Tesseract silently
+        Start-Process -FilePath $tesseractInstaller -ArgumentList "/S", "/D=C:\tools\tesseract" -Wait
+        
+        # Download and extract Poppler
+        $popplerUrl = "https://github.com/oschwartz10612/poppler-windows/releases/download/v25.07.0-0/Release-25.07.0-0.zip"
+        $popplerZip = "C:\tools\poppler.zip"
+        Invoke-WebRequest -Uri $popplerUrl -OutFile $popplerZip
+        
+        # Extract Poppler
+        Expand-Archive -Path $popplerZip -DestinationPath C:\tools\poppler -Force
+        
+        # Add to PATH
+        echo "C:\tools\tesseract" >> $env:GITHUB_PATH
+        echo "C:\tools\poppler\poppler-25.07.0\Library\bin" >> $env:GITHUB_PATH
+        
+        # Set environment variables for your application
+        echo "TESSERACT_FOLDER=C:\tools\tesseract" >> $env:GITHUB_ENV
+        echo "POPPLER_FOLDER=C:\tools\poppler\poppler-25.07.0\Library\bin" >> $env:GITHUB_ENV
+        echo "TESSERACT_DATA_FOLDER=C:\tools\tesseract\tessdata" >> $env:GITHUB_ENV
+        
+        # Verify installation using full paths (since PATH won't be updated in current session)
+        & "C:\tools\tesseract\tesseract.exe" --version
+        & "C:\tools\poppler\poppler-25.07.0\Library\bin\pdftoppm.exe" -v
+    
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pytest pytest-cov reportlab pillow
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+    
+    - name: Run CLI tests
+      run: |
+        cd test
+        python cli_epilog_suite.py
+    
+    - name: Run tests with pytest
+      run: |
+        pytest test/ -v --tb=short

.github/workflows/ci.yml

@@ -0,0 +1,269 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+  #schedule:
+  # Run tests daily at 2 AM UTC
+  #  - cron: '0 2 * * *'
+
+permissions:
+  contents: read
+  actions: read
+  pull-requests: write
+  issues: write
+
+env:
+  PYTHON_VERSION: "3.11"
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install ruff black
+    
+    - name: Run Ruff linter
+      run: ruff check .
+    
+    - name: Run Black formatter check
+      run: black --check .
+
+  test-unit:
+    runs-on: ubuntu-latest
+    env:
+      # Avoid optional VLM/torch import path in tools.run_vlm (not installed in lightweight CI deps)
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    strategy:
+      matrix:
+        python-version: [3.11, 3.12, 3.13]
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Cache pip dependencies
+      uses: actions/cache@v5
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements_lightweight.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_lightweight.txt
+        pip install pytest pytest-cov pytest-html pytest-xdist reportlab pillow
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+        echo "Setup script completed. Checking results:"
+        ls -la doc_redaction/example_data/ || echo "doc_redaction/example_data directory not found"
+    
+    - name: Verify test data files
+      run: |
+        echo "Checking if critical test files exist:"
+        ls -la doc_redaction/example_data/
+        echo "Checking for specific PDF files:"
+        ls -la doc_redaction/example_data/*.pdf || echo "No PDF files found"
+        echo "Checking file sizes:"
+        find doc_redaction/example_data -name "*.pdf" -exec ls -lh {} \;
+    
+    - name: Clean up problematic config files
+      run: |
+        rm -f config*.py || true
+    
+    - name: Run CLI tests
+      run: |
+        cd test
+        python cli_epilog_suite.py
+    
+    - name: Run tests with pytest (JUnit and coverage)
+      run: |
+        pytest test/ -v --tb=short \
+          --junitxml=test-results.xml \
+          --cov=. --cov-config=.coveragerc \
+          --cov-report=xml --cov-report=html --cov-report=term
+    
+    #- name: Upload coverage to Codecov - not necessary
+    #  uses: codecov/codecov-action@v3
+    #  if: matrix.python-version == '3.11'
+    #  with:
+    #    file: ./coverage.xml
+    #    flags: unittests
+    #    name: codecov-umbrella
+    #    fail_ci_if_error: false
+    
+    - name: Upload test results
+      uses: actions/upload-artifact@v6
+      if: always()
+      with:
+        name: test-results-python-${{ matrix.python-version }}
+        path: |
+          test-results.xml
+          htmlcov/
+          coverage.xml
+
+  test-integration:
+    runs-on: ubuntu-latest
+    needs: [lint, test-unit]
+    env:
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_lightweight.txt
+        pip install pytest pytest-cov reportlab pillow
+    
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+        echo "Setup script completed. Checking results:"
+        ls -la doc_redaction/example_data/ || echo "doc_redaction/example_data directory not found"
+    
+    - name: Verify test data files
+      run: |
+        echo "Checking if critical test files exist:"
+        ls -la doc_redaction/example_data/
+        echo "Checking for specific PDF files:"
+        ls -la doc_redaction/example_data/*.pdf || echo "No PDF files found"
+        echo "Checking file sizes:"
+        find doc_redaction/example_data -name "*.pdf" -exec ls -lh {} \;
+    
+    - name: Run integration tests
+      run: |
+        cd test
+        python demo_single_test.py
+    
+    - name: Test CLI help
+      run: |
+        python cli_redact.py --help
+    
+    - name: Test CLI version
+      run: |
+        python -c "import sys; print(f'Python {sys.version}')"
+
+  security:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install safety bandit
+    
+    #- name: Run safety scan - removed as now requires login
+    #  run: |
+    #    safety scan -r requirements.txt
+    
+    - name: Run bandit security check
+      run: |
+        bandit -r . -f json -o bandit-report.json || true
+    
+    - name: Upload security report
+      uses: actions/upload-artifact@v6
+      if: always()
+      with:
+        name: security-report
+        path: bandit-report.json
+
+  build:
+    runs-on: ubuntu-latest
+    needs: [lint, test-unit]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+    
+    - name: Build package
+      run: |
+        python -m build
+    
+    - name: Check package
+      run: |
+        twine check dist/*
+    
+    - name: Upload build artifacts
+      uses: actions/upload-artifact@v6
+      with:
+        name: dist
+        path: dist/

.github/workflows/simple-test.yml

@@ -0,0 +1,74 @@
+name: Simple Test Run
+
+on:
+  push:
+    branches: [ dev ]
+  pull_request:
+    branches: [ dev ]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.12"
+    
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_lightweight.txt
+        pip install pytest pytest-cov reportlab pillow
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+        echo "Setup script completed. Checking results:"
+        ls -la doc_redaction/example_data/ || echo "doc_redaction/example_data directory not found"
+    
+    - name: Verify test data files
+      run: |
+        echo "Checking if critical test files exist:"
+        ls -la doc_redaction/example_data/
+        echo "Checking for specific PDF files:"
+        ls -la doc_redaction/example_data/*.pdf || echo "No PDF files found"
+        echo "Checking file sizes:"
+        find doc_redaction/example_data -name "*.pdf" -exec ls -lh {} \;
+    
+    - name: Run CLI tests
+      run: |
+        cd test
+        python cli_epilog_suite.py
+    
+    - name: Run tests with pytest
+      run: |
+        pytest test/ -v --tb=short

.github/workflows/sync-pi-agent-space.yml

@@ -0,0 +1,64 @@
+name: Sync Pi agent to Hugging Face Space
+
+on:
+  push:
+    branches: [dev]
+    paths:
+      - "agent-redact/**"
+      - "skills/**"
+      - "tools/**"
+      - "intros/**"
+      - "doc_redaction/example_data/**"
+      - "AGENTS.md"
+      - "config/**"
+      - ".github/workflows/sync-pi-agent-space.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  sync-pi-agent-space:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 1
+          lfs: true
+
+      - name: Install Git LFS
+        run: git lfs install
+
+      - name: Materialize example PDFs (Git LFS)
+        run: |
+          git lfs pull --include="doc_redaction/example_data/*.pdf"
+          for f in \
+            doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf \
+            doc_redaction/example_data/graduate-job-example-cover-letter.pdf; do
+            if head -1 "$f" | grep -q "^version https://git-lfs.github.com/spec/v1"; then
+              echo "Example PDF is still an LFS pointer (not materialized): $f" >&2
+              exit 1
+            fi
+          done
+
+      - name: Flatten Pi agent Space tree
+        run: |
+          chmod +x agent-redact/pi-agent/sync_to_space.sh
+          agent-redact/pi-agent/sync_to_space.sh /tmp/pi-agent-space
+
+      - name: Push to Hugging Face Space
+        run: |
+          COMMIT_MSG=$(git log -1 --pretty=%B)
+          echo "Syncing Pi agent Space: seanpedrickcase/agentic_document_redaction"
+          cd /tmp/pi-agent-space
+          git init -b main
+          git config user.name "$HF_USERNAME"
+          git config user.email "$HF_EMAIL"
+          git add .
+          git commit -m "Sync Pi agent Space: $COMMIT_MSG"
+          git remote add hf "https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_USERNAME}/agentic_document_redaction"
+          git push --force hf main
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+          HF_EMAIL: ${{ secrets.HF_EMAIL }}

.github/workflows/sync_to_hf.yml

@@ -0,0 +1,54 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 1      # Only get the latest state
+          lfs: true           # Download actual LFS files so they can be pushed
+
+      - name: Install Git LFS
+        run: git lfs install
+
+      - name: Recreate repo history (single-commit force push)
+        run: |
+          # 1. Capture the message BEFORE we delete the .git folder
+          COMMIT_MSG=$(git log -1 --pretty=%B)
+          echo "Syncing commit message: $COMMIT_MSG"
+
+          # 2. DELETE the .git folder. 
+          # This turns the repo into a standard folder of files.
+          rm -rf .git
+
+          # 3. Re-initialize a brand new git repo
+          git init -b main
+          git config --global user.name "$HF_USERNAME"
+          git config --global user.email "$HF_EMAIL"
+          
+          # 4. Re-install LFS (needs to be done after git init)
+          git lfs install
+          
+          # 5. Add the remote
+          git remote add hf https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$HF_REPO_ID
+          
+          # 6. Add all files
+          # Since this is a fresh init, Git sees EVERY file as "New"
+          git add .
+          
+          # 7. Commit and Force Push
+          git commit -m "Sync: $COMMIT_MSG"
+          git push --force hf main
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+          HF_EMAIL: ${{ secrets.HF_EMAIL }}
+          HF_REPO_ID: ${{ secrets.HF_REPO_ID }}

.github/workflows/sync_to_hf_zero_gpu.yml

@@ -0,0 +1,59 @@
+name: Sync to Hugging Face hub Zero GPU
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  sync-to-hub-zero-gpu:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 1      # Only get the latest state
+          lfs: true           # Download actual LFS files so they can be pushed
+
+      - name: Install Git LFS
+        run: git lfs install
+
+      # HF Spaces read Space config from README.md front matter. The repo README
+      # targets GitHub (e.g. docker); patch only this CI checkout before HF push.
+      - name: Apply HF Zero GPU Space README front matter
+        run: python3 tools/apply_hf_zero_gpu_readme_frontmatter.py
+
+      - name: Recreate repo history (single-commit force push)
+        run: |
+          # 1. Capture the message BEFORE we delete the .git folder
+          COMMIT_MSG=$(git log -1 --pretty=%B)
+          echo "Syncing commit message: $COMMIT_MSG"
+
+          # 2. DELETE the .git folder. 
+          # This turns the repo into a standard folder of files.
+          rm -rf .git
+
+          # 3. Re-initialize a brand new git repo
+          git init -b main
+          git config --global user.name "$HF_USERNAME"
+          git config --global user.email "$HF_EMAIL"
+          
+          # 4. Re-install LFS (needs to be done after git init)
+          git lfs install
+          
+          # 5. Add the remote
+          git remote add hf https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$HF_REPO_ID_ZERO_GPU
+          
+          # 6. Add all files
+          # Since this is a fresh init, Git sees EVERY file as "New"
+          git add .
+          
+          # 7. Commit and Force Push
+          git commit -m "Sync: $COMMIT_MSG"
+          git push --force hf main
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+          HF_EMAIL: ${{ secrets.HF_EMAIL }}
+          HF_REPO_ID_ZERO_GPU: ${{ secrets.HF_REPO_ID_ZERO_GPU }}

.gitignore

@@ -0,0 +1,75 @@
+*.url
+*.ipynb
+*.pyc
+*.qmd
+*.json.bak.*
+_quarto.yml
+quarto_site/*
+src/*
+redaction_deps/*
+.venv/*
+examples/*
+processing/*
+input/*
+output/*
+tools/__pycache__/*
+old_code/*
+tesseract/*
+poppler/*
+build/*
+dist/*
+build_deps/*
+logs/*
+usage/*
+feedback/*
+config/*
+!config/agent.env.example
+!config/docker_app_config.env.example
+!config/app_config.env.example
+workspace/*
+user_guide/*
+_extensions/*
+doc_redaction.egg-info/*
+.venv_pypi_test/*
+cdk/config/*
+!cdk/config/app_config.env.example
+!cdk/config/lambda/
+cdk/config/lambda/*
+!cdk/config/lambda/lambda_function.py
+!cdk/config/headless_s3_seed/
+cdk/config/headless_s3_seed/*
+!cdk/config/headless_s3_seed/input/
+cdk/config/headless_s3_seed/input/*
+!cdk/config/headless_s3_seed/input/config/
+cdk/config/headless_s3_seed/input/config/*
+!cdk/config/headless_s3_seed/input/config/example_headless_env_file.env
+cdk/cdk.out/*
+cdk/archive/*
+tld/*
+tmp/*
+docs/*
+.pi/*
+cdk.out/*
+cdk.json
+cdk.context.json
+precheck.context.json
+.quarto/*
+/.quarto/
+/_site/
+test/config/*
+test/feedback/*
+test/input/*
+test/logs/*
+test/output/*
+test/tmp/*
+test/usage/*
+.ruff_cache/*
+model_cache/*
+sanitized_file/*
+src/doc_redaction.egg-info/*
+docker_compose/*
+**/*.quarto_ipynb
+skills/example_prompts/*
+.pi/sessions/
+agent-redact/pi/agent/sessions/
+agent-redact/RedactionAgent/*

AGENTS.md

@@ -0,0 +1,113 @@
+# AGENTS.md
+
+Context for AI coding agents working on **doc_redaction** (PII redaction for PDFs, images, Word, and tabular files). Human-oriented docs: [README.md](README.md). User guide: [doc_redaction user guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html).
+
+## Project overview
+
+- **Stack**: Python 3.10+, Gradio UI ([app.py](app.py)), optional FastAPI when `RUN_FASTAPI` is enabled, AWS/LLM integrations via [tools/config.py](tools/config.py) and env files under `config/`.
+- **License**: AGPL-3.0-only (see [pyproject.toml](pyproject.toml)). Respect license terms when adding dependencies.
+- **Accuracy**: Outputs are not guaranteed complete; downstream use should assume **human review** of redacted material.
+
+## Cursor skills: redaction workflow (optional)
+
+For agents operating the deployed app (Gradio Client, review CSV, `/review_apply`), these repo-local playbooks are a suggested ladder:
+
+0. **[`skills/doc-redaction-task-prompt/TASK_PROMPT_TEMPLATE.md`](skills/doc-redaction-task-prompt/TASK_PROMPT_TEMPLATE.md)** — copy-paste user task prompt (Pass 1 default, Pass 2 gated); **user redaction requirements go at the end of the prompt**.
+1. **[`skills/doc-redaction-app/SKILL.md`](skills/doc-redaction-app/SKILL.md)** — first-pass redaction (`/doc_redact` / `/redact_document`) and downloading artifacts.
+2. **[`skills/doc-redact-page-review/SKILL.md`](skills/doc-redact-page-review/SKILL.md)** — after outputs exist: **parallel per-page** child agents, merge into one full-document `*_review_file.csv`, **single** `/review_apply` from the parent.
+3. **[`skills/doc-redaction-modifications/SKILL.md`](skills/doc-redaction-modifications/SKILL.md)** — CSV mechanics, `preview_redaction_boxes`, `/review_apply` patterns, verification, VLM and PyMuPDF fallbacks (single-thread edits and the **technical** reference for page-review children).
+
+## Setup
+
+1. **System**: Install **Tesseract** and **Poppler** (required for OCR/PDF). See [README.md](README.md) (Windows/Linux sections).
+2. **Python**: Create a venv, then install the project (e.g. `pip install -e ".[dev]"` or follow README).
+3. **Configuration**: Copy or edit environment/config as described in README / `config/` (e.g. `app_config.env`). Do not commit secrets.
+
+## Run locally
+
+- Gradio/FastAPI entrypoint is [app.py](app.py). With FastAPI enabled, typical pattern is `uvicorn app:app --host 0.0.0.0 --port 7860` (exact host/port from your config).
+- OpenAPI docs: `/docs` when the FastAPI app is mounted.
+
+## Tests
+
+- Run from repo root: `pytest` (optional: `pytest test/`).
+- Fix failures related to your changes before opening a PR.
+
+## Line order (local OCR and simple text extraction)
+
+Multi-column layouts use shared logic in [`tools/ocr_reading_order.py`](tools/ocr_reading_order.py). Controlled by **`LOCAL_OCR_READING_ORDER`** (`column` default, `legacy` for previous top-left behaviour).
+
+### Local OCR (Paddle/Tesseract)
+
+Word boxes are merged into line-level CSV rows in [`combine_ocr_results`](tools/custom_image_analyser_engine.py).
+
+- **`column`**: detect text columns, assign line numbers down each column left-to-right; full-width lines (headers) first. Stops cross-column merging that produced wide erroneous lines on multi-column PDFs. **Auto-fallback**: the page is treated as single-column unless a *consecutive cluster* of gutter rows (y-gap between adjacent rows ≤ `OCR_COLUMN_MAX_CONSECUTIVE_GUTTER_GAP`, default `0.06` of page height) has ≥ `OCR_COLUMN_MIN_GUTTER_ROWS` (default `3`) rows **and** the cluster's topmost row is above the footer zone (`OCR_COLUMN_FOOTER_ZONE_FRACTION`, default `0.75`). This prevents isolated header bands (logo | title, 1 gutter row), signature-only blocks at the page bottom (cluster starts at y ≥ 0.75), or the combination of both, from forcing column mode on the single-column body text between them.
+- **`PADDLE_PRESERVE_LINE_BOXES=True`** or **`CONVERT_LINE_TO_WORD_LEVEL=False`** with Paddle: keep Paddle line boxes (skip word split + regrouping); line numbers still use column reading order.
+
+### Simple text extraction (PyMuPDF)
+
+[`redact_text_pdf`](tools/file_redaction.py) → [`process_page_to_structured_ocr_pymupdf`](tools/file_redaction.py) calls [`reorder_structured_text_lines`](tools/ocr_reading_order.py) after collecting lines, using **`page.mediabox`** width/height for full-span header detection.
+
+`reorder_structured_text_lines` now mirrors `build_line_groups` (local OCR route):
+
+1. **Column-aware sort** (`sort_reading_order` / `assign_layout_boxes` / `detect_column_split_xpoints`) — or legacy top-left for single-column pages.
+2. **Y-band grouping** (`group_into_lines`) — merges any same-row PyMuPDF lines that were emitted as separate objects (e.g. mixed-font spans) and splits horizontally-disparate boxes via `_finalize_line`.  *Column mode only.*
+3. **Secondary sub-column pass** (`_reorder_lines_column_major`) — ensures correct column-major order when sub-columns sit within a single macro-column.  *Column mode only.*
+4. When a group contains more than one box, constituent boxes are **merged** into a single `OCRResult` (union bbox, joined text, concatenated chars/words).
+
+In single-column / legacy mode only step 1 is applied; PyMuPDF lines are pre-formed so no merging is needed.
+
+### Tunables (both routes)
+
+`OCR_FULL_SPAN_WIDTH_RATIO`, `OCR_COLUMN_GAP_MIN_FRACTION`, `OCR_COLUMN_GUTTER_MIN_FRACTION`, `OCR_COLUMN_SUBGUTTER_MIN_FRACTION` (default `0.015` — fine-grained gutter scan in `assign_layout_boxes`; lower = detects narrower sub-column boundaries), `OCR_COLUMN_MIN_GUTTER_ROWS`, `OCR_COLUMN_MAX_BOX_HEIGHT_RATIO`, `OCR_COLUMN_MAX_CONSECUTIVE_GUTTER_GAP`, `OCR_COLUMN_FOOTER_ZONE_FRACTION`, `OCR_LINE_SPLIT_GAP_FRACTION` (default 0.025 — horizontal gap fraction that forces a line split; must be below the narrowest column gutter, ~0.030 for two-page spreads; also used as the gap threshold for the secondary sub-column sort in `build_line_groups`), `OCR_LINE_Y_THRESHOLD_FRACTION` (default 0.013 — row-alignment tolerance as a fraction of page height; reduced from 0.015 to correctly separate tightly-set 10 pt body text whose row spacing is ~0.014), `OCR_LINE_Y_THRESHOLD_MIN_PX`.
+
+**Sub-column ordering** (`build_line_groups`): after the primary word-level column sort, a second pass (`_reorder_lines_column_major`) clusters the produced line groups by their leftmost x-position using `OCR_LINE_SPLIT_GAP_FRACTION` as the gap threshold. This ensures that adjacent narrow sub-columns whose word-level centre gap is below `column_gap_threshold` (e.g. two columns on a spread where each page is already one macro-column) are still output in left-to-right column-major order rather than interleaved by y-position.
+
+**Fine-grained gutter-based column assignment** (`assign_layout_boxes`): before falling back to centre-gap clustering, `detect_column_split_xpoints` scans the page for structural gutters at the finer `OCR_COLUMN_SUBGUTTER_MIN_FRACTION` threshold (default 0.015). Each qualifying gutter cluster produces a `(split_x, y_min)` pair — the split point is only applied to boxes whose `top ≥ y_min`, preventing a narrow sub-column gutter (visible only in the lower two-column section) from mis-splitting a full-width introductory paragraph that sits above it. This correctly separates narrow adjacent columns (e.g. 1.9 % gutter on a two-page spread) without fragmenting full-width headings or paragraphs.
+
+Changing line order affects PII page text, duplicate-page detection, and review CSV line indices on multi-column documents; re-review after upgrading.
+
+## Agentic / programmatic access (two surfaces)
+
+### 1. FastAPI Agent API (recommended for LLM agents: small JSON bodies)
+
+When `RUN_FASTAPI` is true, routes are mounted under **`/agent`** ([agent_routes.py](agent_routes.py)).
+
+- **Catalog**: `GET /agent/operations` — maps each Gradio `api_name` to an HTTP path and notes whether the route is implemented via CLI or returns HTTP 501 for Gradio-only flows.
+- **Implemented POST routes** (CLI- or [tools/simplified_api.py](tools/simplified_api.py)-backed where noted):  
+  `redact_document`, `redact_data`, `find_duplicate_pages`, `find_duplicate_tabular`, `summarise_document`, `combine_review_pdfs`, `combine_review_csvs`, `export_review_redaction_overlay`, `export_review_page_ocr_visualisation`, `apply_review_redactions`, **`verify_redaction_coverage`** (Pass 1 QA: `must_redact` / `must_not_redact` regex lists, optional `redacted_pdf_path`, optional `auto_prune_suspicious` + `pruned_output_path`; returns `pass_strict`, `pass_with_cleanup`, `pages_flagged_for_vlm`, `pages_needing_csv_cleanup`), **`word_level_ocr_text_search`** (headless word OCR search with optional review-box overlap flags).
+
+**Optional post-redaction Pass 1 QA (main app / CLI):** When `POST_REDACT_PASS1_QA=True` in [`tools/config.py`](tools/config.py) (or `config/app_config.env`), initial redaction emits `*_coverage_report.json` beside the review CSV and optionally `*_review_file_pruned.csv` (sibling, when `POST_REDACT_PASS1_AUTO_PRUNE=True`). Uses deny/allow lists and/or `POST_REDACT_PASS1_MUST_REDACT_PATH` / `POST_REDACT_PASS1_MUST_NOT_REDACT_PATH`. CLI overrides: `--post-redact-pass1-qa`, `--post-redact-pass1-auto-prune`. This is pre-review-apply sanity QA only — agent Pass 1 (policy edits + `/review_apply`) remains separate.  
+  Note: on Gradio ([app.py](app.py)), the Review-tab visual exports use `api_name` **`page_redaction_review_image`** and **`page_ocr_review_image`**; the **`/agent`** routes above keep the explicit `export_review_*` names for the same operations.
+- **Gradio-only stubs** (501 + JSON hint): `load_and_prepare_documents_or_data`.
+- **Auth**: If `AGENT_API_KEY` is set in the environment, send header `X-Agent-API-Key` with that value.
+- **Paths**: Inputs must resolve to files under the repo root, `INPUT_FOLDER`, or `OUTPUT_FOLDER` (see router validation).
+
+Implementation uses **`cli_redact.main(direct_mode_args=...)`** where a CLI task exists (same behaviour as [cli_redact.py](cli_redact.py)); `apply_review_redactions` calls [tools/simplified_api.py](tools/simplified_api.py) instead.
+
+### 2. Gradio Client API (e.g. Hugging Face Spaces)
+
+For remote Spaces or any Gradio deployment exposing the HTTP API:
+
+- **Schema**: `GET https://<host>/gradio_api/info`
+- **Call**: `POST https://<host>/gradio_api/call/{api_name}` with body `{"data":[...]}` (argument order matches the named endpoint’s component list).
+- **Poll**: `GET https://<host>/gradio_api/call/{api_name}/{event_id}`
+- **Hugging Face**: `Authorization: Bearer $HF_TOKEN`
+
+Named `api_name` values in this app include: `redact_document`, `load_and_prepare_documents_or_data`, `apply_review_redactions`, **`doc_redact`** (simple `gr.api`: one PDF/image + optional OCR/PII knobs; returns `(output_paths, message)`; `api_name='/doc_redact'`; parameters include `document_file`, `redact_entities`, `output_dir`, `ocr_method`, `pii_method`, `allow_list`, `deny_list`, `page_min`, `page_max`, **`handwrite_signature_checkbox`** — AWS Textract extraction options such as `Extract handwriting` / `Extract signatures`), **`review_apply`** (simple `gr.api`: PDF + `*_review_file.csv`; returns `(output_paths, message)`; `api_name='/review_apply'`), **`preview_boxes`** (simple `gr.api`: PDF + `*_review_file.csv`; renders proposed boxes onto the original PDF and returns `(zip_path, message)` — use to verify coordinates *before* calling `review_apply`, no redaction applied; `api_name='/preview_boxes'`), **`pdf_summarise`** (simple `gr.api`: PDF + optional summarisation/OCR knobs; returns `(output_paths, status_message, summary_text)`; `api_name='/pdf_summarise'`), **`tabular_redact`** (simple `gr.api`: one tabular file (CSV/XLSX/Parquet/DOCX) + optional knobs; returns `(output_paths, message)`; `api_name='/tabular_redact'`), **`page_redaction_review_image`** (short review overlay export; `api_name='/page_redaction_review_image'`), **`page_ocr_review_image`** (short OCR visualisation export; `api_name='/page_ocr_review_image'`), `word_level_ocr_text_search`, `redact_data`, `find_duplicate_pages`, `find_duplicate_tabular`, `summarise_document`, `combine_review_csvs`, `combine_review_pdfs`. The matching **`POST /agent`** names for those two visual exports are `export_review_redaction_overlay` and `export_review_page_ocr_visualisation` (§1). Many endpoints require **many positional arguments** (full Gradio state); prefer the short `gr.api` routes above or **`POST /agent/apply_review_redactions`** where applicable instead of building the full `data` array from `/gradio_api/info`.
+
+## CLI parity
+
+For scripting and tests, `python cli_redact.py` with flags is authoritative; programmatic merges use `get_cli_default_args_dict()` in [cli_redact.py](cli_redact.py).
+
+## Security and data handling
+
+- Do not commit API keys, tokens, or customer data.
+- Treat paths as untrusted outside validated roots (see [tools/secure_path_utils.py](tools/secure_path_utils.py)).
+- Optional `instruction` / LLM fields must not be passed into shell or unconstrained config keys.
+
+## Conventions for PRs
+
+- Keep changes focused; avoid drive-by refactors.
+- Match existing naming and patterns in [app.py](app.py) and [tools/](tools/).
+- Update tests when behaviour changes; run `pytest` before merge.

Dockerfile

@@ -0,0 +1,235 @@
+# Stage 1: Build dependencies and download models
+FROM public.ecr.aws/docker/library/python:3.12.13-slim-trixie AS builder
+
+# Install system dependencies
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y --no-install-recommends \
+        g++ \
+        make \
+        cmake \
+        unzip \
+        libcurl4-openssl-dev \
+        git \
+    && pip install --upgrade pip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /src
+
+COPY requirements_lightweight.txt .
+
+RUN pip install --verbose --no-cache-dir --target=/install -r requirements_lightweight.txt && rm requirements_lightweight.txt
+
+ARG INSTALL_GRADIO_MCP=False
+ENV INSTALL_GRADIO_MCP=${INSTALL_GRADIO_MCP}
+
+RUN if [ "$INSTALL_GRADIO_MCP" = "True" ]; then \
+    pip install --verbose --no-cache-dir --force-reinstall --target=/install "gradio[mcp]>=6.16.0"; \
+fi
+
+# Optionally install PaddleOCR if the INSTALL_PADDLEOCR environment variable is set to True. Note that GPU-enabled PaddleOCR is unlikely to work in the same environment as a GPU-enabled version of PyTorch, so it is recommended to install PaddleOCR as a CPU-only version if you want to use GPU-enabled PyTorch.
+
+ARG INSTALL_PADDLEOCR=False
+ENV INSTALL_PADDLEOCR=${INSTALL_PADDLEOCR}
+
+ARG PADDLE_GPU_ENABLED=False
+ENV PADDLE_GPU_ENABLED=${PADDLE_GPU_ENABLED}
+
+RUN if [ "$INSTALL_PADDLEOCR" = "True" ] && [ "$PADDLE_GPU_ENABLED" = "False" ]; then \
+    pip install --verbose --no-cache-dir --target=/install "protobuf<=7.34.0" && \
+    pip install --verbose --no-cache-dir --target=/install "paddlepaddle<=3.2.1" && \
+    pip install --verbose --no-cache-dir --target=/install "paddleocr<=3.7.0"; \
+elif [ "$INSTALL_PADDLEOCR" = "True" ] && [ "$PADDLE_GPU_ENABLED" = "True" ]; then \
+    pip install --verbose --no-cache-dir --target=/install "protobuf<=7.34.0" && \
+    pip install --verbose --no-cache-dir --target=/install "paddlepaddle<=3.2.1" && \
+    pip install --verbose --no-cache-dir --target=/install "paddleocr<=3.7.0" && \
+    pip install --verbose --no-cache-dir --target=/install "torch<=2.10.0" --index-url https://download.pytorch.org/whl/cu129 && \
+    pip install --verbose --no-cache-dir --target=/install "torchvision<=0.25.0" --index-url https://download.pytorch.org/whl/cu129 && \
+    pip install --verbose --no-cache-dir --target=/install "transformers<=5.12.0"; \
+fi
+
+ARG INSTALL_VLM=False
+ENV INSTALL_VLM=${INSTALL_VLM}
+
+ARG TORCH_GPU_ENABLED=False
+ENV TORCH_GPU_ENABLED=${TORCH_GPU_ENABLED}
+
+# Optionally install VLM/LLM packages if the INSTALL_VLM environment variable is set to True.
+RUN if [ "$INSTALL_VLM" = "True" ] && [ "$TORCH_GPU_ENABLED" = "False" ]; then \
+    pip install --verbose --no-cache-dir --target=/install \
+    "torch==2.10.0+cpu" \
+    "torchvision==0.25.0+cpu" \
+    "transformers<=5.12.0" \
+    "accelerate<=1.13.0" \
+    "bitsandbytes<=0.49.2" \
+    "sentencepiece<=0.2.1" \
+    --extra-index-url https://download.pytorch.org/whl/cpu; \
+elif [ "$INSTALL_VLM" = "True" ] && [ "$TORCH_GPU_ENABLED" = "True" ]; then \
+    pip install --verbose --no-cache-dir --target=/install "torch<=2.10.0" --index-url https://download.pytorch.org/whl/cu129 && \
+    pip install --verbose --no-cache-dir --target=/install "torchvision<=0.25.0" --index-url https://download.pytorch.org/whl/cu129 && \
+    pip install --verbose --no-cache-dir --target=/install \
+        "transformers<=5.12.0" \
+        "accelerate<=1.13.0" \
+        "bitsandbytes<=0.49.2" \
+        "sentencepiece<=0.2.1" && \
+    pip install --verbose --no-cache-dir --target=/install "optimum<=2.1.0" && \
+    pip install --verbose --no-cache-dir --target=/install  https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.8cxx11abiTRUE-cp312-cp312-linux_x86_64.whl && \
+    pip install --verbose --no-cache-dir --target=/install  https://github.com/ModelCloud/GPTQModel/releases/download/v5.8.0/gptqmodel-5.8.0+cu128torch2.8-cp312-cp312-linux_x86_64.whl; \
+fi
+
+# ===================================================================
+# Stage 2: A common base for both Lambda and Gradio
+# ===================================================================
+FROM public.ecr.aws/docker/library/python:3.12.13-slim-trixie AS base
+
+# MUST re-declare ARGs in every stage where they are used in RUN commands
+ARG TORCH_GPU_ENABLED=False
+ARG PADDLE_GPU_ENABLED=False
+
+ENV TORCH_GPU_ENABLED=${TORCH_GPU_ENABLED}
+ENV PADDLE_GPU_ENABLED=${PADDLE_GPU_ENABLED}
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    tesseract-ocr \
+    poppler-utils \
+    libgl1 \
+    libglib2.0-0 && \
+    if [ "$TORCH_GPU_ENABLED" = "True" ] || [ "$PADDLE_GPU_ENABLED" = "True" ]; then \
+        apt-get install -y --no-install-recommends libgomp1; \
+    fi && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV APP_HOME=/home/user
+
+# Set env variables for Gradio & other apps
+ENV GRADIO_TEMP_DIR=/tmp/gradio_tmp/ \
+    MPLCONFIGDIR=/tmp/matplotlib_cache/ \
+    GRADIO_OUTPUT_FOLDER=$APP_HOME/app/output/ \
+    GRADIO_INPUT_FOLDER=$APP_HOME/app/input/ \
+    FEEDBACK_LOGS_FOLDER=$APP_HOME/app/feedback/ \
+    ACCESS_LOGS_FOLDER=$APP_HOME/app/logs/ \
+    USAGE_LOGS_FOLDER=$APP_HOME/app/usage/ \
+    CONFIG_FOLDER=$APP_HOME/app/config/ \
+    XDG_CACHE_HOME=/tmp/xdg_cache/user_1000 \
+    TESSERACT_DATA_FOLDER=/usr/share/tessdata \
+    GRADIO_SERVER_NAME=0.0.0.0 \
+    GRADIO_SERVER_PORT=7860 \
+    PATH=$APP_HOME/.local/bin:$PATH \
+    PYTHONPATH=$APP_HOME/app \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    GRADIO_ALLOW_FLAGGING=never \
+    GRADIO_NUM_PORTS=1 \    
+    GRADIO_ANALYTICS_ENABLED=False
+
+# Copy Python packages from the builder stage
+COPY --from=builder /install /usr/local/lib/python3.12/site-packages/
+COPY --from=builder /install/bin /usr/local/bin/
+
+# Reinstall protobuf into the final site-packages. Builder uses multiple `pip install --target=/install`
+# passes; that can break the `google` namespace so `google.protobuf` is missing and Paddle fails at import.
+RUN pip install --no-cache-dir "protobuf<=7.34.0"
+
+# English pipeline is not a normal PyPI dependency; bundle it in the image so runtime works offline.
+# Placed before COPY app code so application changes do not invalidate this layer.
+RUN python -m spacy download en_core_web_lg
+
+# Copy your application code and entrypoint
+COPY . ${APP_HOME}/app
+COPY entrypoint.sh ${APP_HOME}/app/entrypoint.sh
+# Fix line endings and set execute permissions
+RUN sed -i 's/\r$//' ${APP_HOME}/app/entrypoint.sh \
+    && chmod +x ${APP_HOME}/app/entrypoint.sh
+
+WORKDIR ${APP_HOME}/app
+
+# ===================================================================
+# FINAL Stage 3: The Lambda Image (runs as root for simplicity)
+# ===================================================================
+FROM base AS lambda
+# Set runtime ENV for Lambda mode
+ENV APP_MODE=lambda
+ENTRYPOINT ["/home/user/app/entrypoint.sh"]
+CMD ["lambda_entrypoint.lambda_handler"]
+
+# ===================================================================
+# FINAL Stage 4: The Gradio Image (runs as a secure, non-root user)
+# ===================================================================
+FROM base AS gradio
+# Set runtime ENV for Gradio mode
+ENV APP_MODE=gradio
+
+# Create non-root user
+RUN useradd -m -u 1000 user
+
+# Create the base application directory and set its ownership
+RUN mkdir -p ${APP_HOME}/app && chown user:user ${APP_HOME}/app
+
+# Create required sub-folders within the app directory and set their permissions
+# This ensures these specific directories are owned by 'user'
+RUN mkdir -p \
+    ${APP_HOME}/app/output \
+    ${APP_HOME}/app/input \
+    ${APP_HOME}/app/logs \
+    ${APP_HOME}/app/usage \
+    ${APP_HOME}/app/feedback \
+    ${APP_HOME}/app/config \
+    && chown user:user \
+    ${APP_HOME}/app/output \
+    ${APP_HOME}/app/input \
+    ${APP_HOME}/app/logs \
+    ${APP_HOME}/app/usage \
+    ${APP_HOME}/app/feedback \
+    ${APP_HOME}/app/config \
+    && chmod 755 \
+    ${APP_HOME}/app/output \
+    ${APP_HOME}/app/input \
+    ${APP_HOME}/app/logs \
+    ${APP_HOME}/app/usage \
+    ${APP_HOME}/app/feedback \
+    ${APP_HOME}/app/config 
+
+# Now handle the /tmp and /var/tmp directories and their subdirectories, paddle, spacy, tessdata
+RUN mkdir -p /tmp/gradio_tmp /tmp/tld /tmp/matplotlib_cache /tmp /var/tmp ${XDG_CACHE_HOME} \
+    && chown user:user /tmp /var/tmp /tmp/gradio_tmp /tmp/tld /tmp/matplotlib_cache ${XDG_CACHE_HOME} \
+    && chmod 1777 /tmp /var/tmp /tmp/gradio_tmp /tmp/tld /tmp/matplotlib_cache \
+    && chmod 700 ${XDG_CACHE_HOME} \
+    && mkdir -p ${APP_HOME}/.paddlex \
+    && chown user:user ${APP_HOME}/.paddlex \
+    && chmod 755 ${APP_HOME}/.paddlex \
+    && mkdir -p ${APP_HOME}/.local/share/spacy/data \
+    && chown user:user ${APP_HOME}/.local/share/spacy/data \
+    && chmod 755 ${APP_HOME}/.local/share/spacy/data \
+    && mkdir -p /usr/share/tessdata \
+    && chown user:user /usr/share/tessdata \
+    && chmod 755 /usr/share/tessdata
+
+# Fix apply user ownership to all files in the home directory
+RUN chown -R user:user /home/user
+
+# Set permissions for Python executable
+RUN chmod 755 /usr/local/bin/python
+
+# Declare volumes (NOTE: runtime mounts will override permissions — handle with care)
+VOLUME ["/tmp/matplotlib_cache"]
+VOLUME ["/tmp/gradio_tmp"]
+VOLUME ["/tmp/tld"]
+VOLUME ["/home/user/app/output"]
+VOLUME ["/home/user/app/input"]
+VOLUME ["/home/user/app/logs"]
+VOLUME ["/home/user/app/usage"]
+VOLUME ["/home/user/app/feedback"]
+VOLUME ["/home/user/app/config"]
+VOLUME ["/home/user/.paddlex"]
+VOLUME ["/home/user/.local/share/spacy/data"]
+VOLUME ["/usr/share/tessdata"]
+VOLUME ["/tmp"]
+VOLUME ["/var/tmp"]
+
+USER user
+
+EXPOSE $GRADIO_SERVER_PORT
+
+ENTRYPOINT ["/home/user/app/entrypoint.sh"]
+CMD ["python", "app.py"]

LICENSE

@@ -0,0 +1,661 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

MANIFEST.in

@@ -0,0 +1,4 @@
+recursive-include doc_redaction/assets *.png
+recursive-include doc_redaction/example_data *
+recursive-include intros *.txt
+

README.md

@@ -0,0 +1,367 @@
+---
+title: Document redaction
+emoji: 📝
+colorFrom: blue
+colorTo: yellow
+sdk: docker
+app_file: app.py
+pinned: true
+license: agpl-3.0
+short_description: OCR / redact PDF documents and tabular data
+---
+# Document redaction (doc_redaction)
+
+<a href="https://pypi.org/project/doc-redaction/" target="_blank"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/doc-redaction"></a> 
+
+Redact personally identifiable information (PII) from documents (PDF, PNG, JPG), Word files (DOCX), or tabular data (XLSX/CSV/Parquet). Please see the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for a full walkthrough of all the features in the app.
+
+---
+
+## 🚀 Quick Start - Installation and first run
+
+Follow these instructions to get the document redaction application running on your local machine.
+
+### 1. Installation
+
+#### Option 1 - Recommended: Install from source repo
+
+Clone the repository and install in editable mode:
+
+```bash
+git clone https://github.com/seanpedrick-case/doc_redaction.git
+cd doc_redaction
+pip install -e .
+```
+
+##### Install extras (Paddle or Transformers/Torch VLM)
+
+To install with PaddleOCR (with a transformers backend as of v2.4.0):
+
+```bash
+pip install -e ".[paddle]"
+```
+
+
+If you want to run VLMs / LLMs with the transformers package:
+
+```bash
+pip install -e ".[vlm]"
+```
+
+
+Note that the versions of both PaddleOCR and Torch installed by default are the CPU-only versions. If you want to install the GPU-enabled version of torch, it is advised to install the following version:
+```bash
+pip install torch==2.10.0 torchvision==0.25.0 --index-url https://download.pytorch.org/whl/cu129
+```
+
+#### Option 2 - Install from PyPI
+
+Create a virtual environment (recommended) and install **doc_redaction**.
+
+```bash
+python -m venv venv
+# Windows:
+.\venv\Scripts\activate
+# macOS/Linux:
+source venv/bin/activate
+```
+
+The package is published on PyPI as **`doc-redaction`** (import name **`doc_redaction`**):
+
+```bash
+pip install doc_redaction
+```
+
+Optional extras (same as in `pyproject.toml`). For installing paddleOCR:
+
+```bash
+pip install "doc_redaction[paddle]"
+```
+
+For running VLMs / LLMs with the transformers package:
+
+```bash
+pip install "doc_redaction[vlm]"
+```
+
+For programmatic use (CLI-first API matching Gradio `api_name` routes), see **[Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html)**. The console script **`cli_redact`** is available after install.
+
+**Web UI from a PyPI install:** You *can* start the Gradio UI after `pip install doc_redaction` by running (note that the prerequisites tesseract and poppler will need to be correctly installed following step 2 below):
+
+```bash
+python -m app
+```
+
+**Important: your working directory matters.** When you run `python -m app`, the app treats your *current folder* as the “app folder”:
+
+- It will look for configuration at `config/app_config.env` *relative to the folder you run it from* (and `python -m doc_redaction.install_deps` will also write `config/app_config.env` there).
+- It may create new folders in that location (for example `config/`, `output/`, `input/`, `logs/`, `usage/`, `feedback/`, and temporary/cache folders depending on your settings).
+- The UI example files and bundled assets are packaged with the PyPI install (they live inside the installed `doc_redaction` package). If you run from a “random” directory after a PyPI install, the app can still locate its packaged examples; your working directory mainly affects where `config/`, `input/`, `output/`, logs, and temp folders are created.
+
+In practice, the **smoothest UI experience** (examples, bundled assets, docs links, predictable relative paths) is still usually via a **repository checkout** or **Docker**, but PyPI install is sufficient to launch the UI as long as you run it from a suitable working folder and have the system dependencies available (or run `python -m doc_redaction.install_deps` first).
+
+#### Option 3 - Docker installation
+
+The doc_redaction Redaction app can be installed by using the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) or Docker compose files ([llama.cpp](https://github.com/ggml-org/llama.cpp), [vLLM](https://docs.vllm.ai/en/stable/)) provided in the repo.
+
+##### With Llama.cpp / vLLM inference server
+
+The project now has Docker and Docker compose files available to pair running the Redaction app with local inference servers powered by [llama.cpp](https://github.com/ggml-org/llama.cpp), or [vLLM](https://docs.vllm.ai/en/stable/). Llama.cpp is more flexible than vLLM for low VRAM systems, as Llama.cpp will offload to cpu/system RAM automatically rather than failing as vLLM tends to do.
+
+For Llama.cpp, you can use the [docker-compose_llama.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_llama.yml) file, and for vLLM, you can use the [docker-compose_vllm.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_vllm.yml) file. To run, Docker / Docker Desktop should be installed, and then you can run the commands suggested in the top of the files to run the servers.
+
+You will need ~40 GB of disk space to run everything depending on the model chosen from the compose file. For the vLLM server, you will need 24 GB VRAM. For the Llama.cpp server, 24 GB VRAM is needed to run at full speed, but the n-gpu-layers and n-cpu-moe parameters in the Docker compose file can be adjusted to fit into your system. I would suggest that 8 GB VRAM is needed as a bare minimum for decent inference speed. See the [Unsloth guide](https://unsloth.ai/docs/models/qwen3.5) for more details on working with GGUF files for Qwen 3.5.
+
+##### Without Llama.cpp / vLLM inference server
+
+If you want a working Docker installation without GPU support, you can install from the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) in the repo. A working example of this, with the CPU version of PaddleOCR, can be found on [Hugging Face](https://huggingface.co/spaces/seanpedrickcase/document_redaction). You can adjust the INSTALL_PADDLEOCR, PADDLE_GPU_ENABLED, INSTALL_VLM, and TORCH_GPU_ENABLED config variables to adjust for PaddleOCR and Transformers packages for local VLM support. Note that GPU-enabled PaddleOCR, and GPU-enabled Transformers/Torch often don't work well together, which is one reason why a Llama.cpp/vLLM inference server Docker installation option is provided below.
+
+The main [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) produces two final images via build targets: **`gradio`** (default web UI, non-root user, named volumes for writable paths) and **`lambda`** (AWS Lambda handler). Build examples:
+
+```bash
+docker build -f Dockerfile --target gradio -t doc-redaction-gradio .
+docker build -f Dockerfile --target lambda -t doc-redaction-lambda .
+```
+
+##### Pi agent (agentic redaction)
+
+The [Pi](https://github.com/earendil-works/pi) orchestration UI uses a separate multi-stage image at [agent-redact/pi-agent/Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/agent-redact/pi-agent/Dockerfile). It shares the same Python 3.12 slim base as the main app; a small Node stage installs the `pi` CLI, which is copied into the runtime image.
+
+| Build target | Typical use |
+|--------------|-------------|
+| **`dev`** | Local development with [docker-compose_llama_agentic.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_llama_agentic.yml) — the repo is bind-mounted; only Pi CLI + Python deps are in the image. |
+| **`runtime`** | [Hugging Face Space](https://huggingface.co/spaces/seanpedrickcase/agentic_document_redaction) and AWS ECS — agent code is baked in; runs as non-root `user` with **named volumes** for workspace, uploads, and session dirs (read-only root filesystem friendly). |
+
+Build from the repository root:
+
+```bash
+docker build -f agent-redact/pi-agent/Dockerfile --target dev -t pi-agent-dev .
+docker build -f agent-redact/pi-agent/Dockerfile --target runtime -t pi-agent-runtime .
+```
+
+For llama.cpp + Pi together, see the compose examples at the top of [docker-compose_llama_agentic.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_llama_agentic.yml). Further detail: [agent-redact/README.md](https://github.com/seanpedrick-case/doc_redaction/blob/main/agent-redact/README.md).
+
+#### Option 4 - Installation on AWS with CDK
+
+The repo contains a [CDK folder](https://github.com/seanpedrick-case/doc_redaction/tree/main/cdk), that contains all the files you need to setup and deploy to an AWS environment with CDK. The installation wizard is [cdk_install.py](https://github.com/seanpedrick-case/doc_redaction/blob/main/cdk/cdk_install.py), which provides a number of options to deploy the Document Redaction App to AWS for demonstration or production. CDK-specific notes (including CloudFront CSP/CORS and the post-deploy refresh deploy) are in [cdk/README.md](cdk/README.md). More details on CDK deployment can be found in the [Installation Guide](https://seanpedrick-case.github.io/doc_redaction/src/installation_guide.html).
+
+### 2. Install prerequisites: Tesseract and Poppler
+
+This application relies on two external tools for OCR (Tesseract) and PDF processing (Poppler). If not using a Docker-based deployment, you will need to install them on your system before proceeding. To run the Document Redaction app successfully, these tools need to be installed and either 1. added to PATH, or 2. be in a folder that is directly referenced in the config/app_config.env file with the variables TESSERACT_FOLDER and POPPLER_FOLDER (defined [here](https://github.com/seanpedrick-case/doc_redaction/blob/main/tools/config.py) if you want to see the code). The instructions below will guide you through different ways to install these dependencies.
+
+---
+
+#### Automated dependency setup (recommended)
+
+If you **don’t have admin rights** (or you just want the simplest setup), you can have the project download and configure **Tesseract** and **Poppler** into a local `redaction_deps/` folder inside the doc_redaction folder.
+
+You need the installer script available first, which means either:
+
+- **Repository checkout**: `git clone ...` and run the command from the repo root (recommended for the web UI), or
+- **PyPI install**: `pip install doc_redaction` and run from a writable folder where you want `redaction_deps/` and `config/app_config.env` to be created/updated.
+
+From the repository root (or your chosen working folder) after creating/activating your venv and installing Python requirements:
+
+```bash
+python -m doc_redaction.install_deps
+```
+
+This writes `TESSERACT_FOLDER` / `POPPLER_FOLDER` into `config/app_config.env` so the app can find the binaries without you editing your system PATH.
+
+To just check whether your machine can already see the tools:
+
+```bash
+python -m doc_redaction.install_deps --verify-only
+```
+
+#### **On Windows**
+
+If you don’t use the automated setup above, you can install the dependencies manually by downloading installers and adding the programs to your system's PATH.
+
+1.  **Install Tesseract OCR:**
+    *   Download the installer from the official Tesseract at [UB Mannheim page](https://github.com/UB-Mannheim/tesseract/wiki) (e.g., `tesseract-ocr-w64-setup-v5.X.X...exe`).
+    *   Run the installer.
+    *   **IMPORTANT:** During installation, ensure you select the option to "Add Tesseract to system PATH for all users" or a similar option. This is crucial for the application to find the Tesseract executable.
+
+
+2.  **Install Poppler:**
+    *   Download the latest Poppler binary for Windows. A common source is the [Poppler for Windows](https://github.com/oschwartz10612/poppler-windows) GitHub releases page. Download the `.zip` file (e.g., `poppler-25.07.0-win.zip`).
+    *   Extract the contents of the zip file to a permanent location on your computer, for example, `C:\Program Files\poppler\`.
+    *   You must add the `bin` folder from your Poppler installation to your system's PATH environment variable.
+        *   Search for "Edit the system environment variables" in the Windows Start Menu and open it.
+        *   Click the "Environment Variables..." button.
+        *   In the "System variables" section, find and select the `Path` variable, then click "Edit...".
+        *   Click "New" and add the full path to the `bin` directory inside your Poppler folder (e.g., `C:\Program Files\poppler\poppler-24.02.0\bin`).
+        *   Click OK on all windows to save the changes.
+
+    To verify, open a new Command Prompt and run `tesseract --version` and `pdftoppm -v`. If they both return version information, you have successfully installed the prerequisites.
+---
+
+#### **On Linux (Debian/Ubuntu)**
+
+Open your terminal and run the following command to install Tesseract and Poppler:
+
+```bash
+sudo apt-get update && sudo apt-get install -y tesseract-ocr poppler-utils
+```
+
+#### **On Linux (Fedora/CentOS/RHEL)**
+
+Open your terminal and use the `dnf` or `yum` package manager:
+
+```bash
+sudo dnf install -y tesseract poppler-utils
+```
+---
+
+### 3. Run the Application
+
+With all dependencies installed, you can now start the Gradio application GUI. For a guide on how to use this, please go [here](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html).
+
+```bash
+python app.py
+```
+
+After running the command, the application will start, and you will see a local URL in your terminal (usually `http://127.0.0.1:7860`).
+
+Open this URL in your web browser to use the document redaction tool
+
+#### Command line interface
+
+For example CLI commands, please refer to [this guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html#command-line-interface-cli) or the examples in [cli_redact.py](https://github.com/seanpedrick-case/doc_redaction/blob/main/cli_redact.py#L321)
+
+If you installed from **PyPI**, use the installed console script:
+
+```bash
+cli_redact --help
+```
+
+From a **repository checkout**, you can also run:
+
+```bash
+python cli_redact.py --help
+```
+
+#### Python package commands
+
+For Python examples in using the Python package, please see [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+
+---
+
+
+### 4. ⚙️ Configuration (Optional)
+
+You can customise the application's behavior by creating a configuration file. This allows you to change settings without modifying the source code, such as enabling AWS features, changing logging behavior, or pointing to local Tesseract/Poppler installations. A full overview of all the potential settings you can modify in the app_config.env file can be seen in tools/config.py, with explanation on the documentation website for [the github repo](https://seanpedrick-case.github.io/doc_redaction/)
+
+To get started:
+1.  Copy `config/app_config.env.example` to `config/app_config.env`.
+2.  Modify the values in `config/app_config.env` to suit your needs. The application will automatically load these settings on startup.
+
+If you do not create this file, the application will run with default settings.
+
+#### Configuration Breakdown
+
+Here is an overview of the most important settings, separated by whether they are for local use or require AWS.
+
+---
+
+#### **Local & General Settings (No AWS Required)**
+
+These settings are useful for all users, regardless of whether you are using AWS.
+
+*   `TESSERACT_FOLDER` / `POPPLER_FOLDER`
+    *   Use these if you installed Tesseract or Poppler to a custom location on **Windows** and did not add them to the system PATH.
+    *   Provide the path to the respective installation folders (for Poppler, point to the `bin` sub-directory).
+    *   **Examples:** `POPPLER_FOLDER=C:/Program Files/poppler-24.02.0/bin/` `TESSERACT_FOLDER=tesseract/`
+
+*   `TESSERACT_DATA_FOLDER`
+    *   If Tesseract runs but you see an error like `Error opening data file ./eng.traineddata` or `Tesseract couldn't load any languages`, this is usually because it can't find the `tessdata/` language files.
+    *   Set this to the folder that contains `eng.traineddata` (typically a `tessdata` directory).
+    *   **Examples (Windows):** `TESSERACT_DATA_FOLDER=C:/Program Files/Tesseract-OCR/tessdata`
+
+*   `SHOW_LANGUAGE_SELECTION=True`
+    *   Set to `True` to display a language selection dropdown in the UI for OCR processing.
+
+*   `DEFAULT_LOCAL_OCR_MODEL=tesseract`"
+    *   Choose the backend for local OCR. Options are `tesseract`, `paddle`, or `hybrid`. "Tesseract" is the default, and is recommended. "hybrid-paddle" is a combination of the two - first pass through the redactions will be done with Tesseract, and then a second pass will be done with PaddleOCR on words with low confidence. "paddle" will only return whole line text extraction, and so will only work for OCR, not redaction. 
+
+*   `SESSION_OUTPUT_FOLDER=False`
+    *   If `True`, redacted files will be saved in unique subfolders within the `output/` directory for each session.
+
+*   `DISPLAY_FILE_NAMES_IN_LOGS=False`
+    *   For privacy, file names are not recorded in usage logs by default. Set to `True` to include them.
+
+---
+
+#### **AWS-Specific Settings**
+
+These settings are only relevant if you intend to use AWS services like Textract for OCR and Comprehend for PII detection.
+
+*   `RUN_AWS_FUNCTIONS=True`
+    *   **This is the master switch.** You must set this to `True` to enable any AWS functionality. If it is `False`, all other AWS settings will be ignored.
+
+*   **UI Options:**
+    *   `SHOW_AWS_TEXT_EXTRACTION_OPTIONS=True`: Adds "AWS Textract" as an option in the text extraction dropdown.
+    *   `SHOW_AWS_PII_DETECTION_OPTIONS=True`: Adds "AWS Comprehend" as an option in the PII detection dropdown.
+
+*   **Core AWS Configuration:**
+    *   `AWS_REGION=example-region`: Set your AWS region (e.g., `us-east-1`).
+    *   `DOCUMENT_REDACTION_BUCKET=example-bucket`: The name of the S3 bucket the application will use for temporary file storage and processing.
+
+*   **AWS Logging:**
+    *   `SAVE_LOGS_TO_DYNAMODB=True`: If enabled, usage and feedback logs will be saved to DynamoDB tables.
+    *   `ACCESS_LOG_DYNAMODB_TABLE_NAME`, `USAGE_LOG_DYNAMODB_TABLE_NAME`, etc.: Specify the names of your DynamoDB tables for logging.
+
+*   **Advanced AWS Textract Features:**
+    *   `SHOW_WHOLE_DOCUMENT_TEXTRACT_CALL_OPTIONS=True`: Enables UI components for large-scale, asynchronous document processing via Textract.
+    *   `TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET=example-bucket-output`: A separate S3 bucket for the final output of asynchronous Textract jobs.
+    *   `LOAD_PREVIOUS_TEXTRACT_JOBS_S3=True`: If enabled, the app will try to load the status of previously submitted asynchronous jobs from S3.
+
+*   **Cost Tracking (for internal accounting):**
+    *   `SHOW_COSTS=True`: Displays an estimated cost for AWS operations. Can be enabled even if AWS functions are off.
+    *   `GET_COST_CODES=True`: Enables a dropdown for users to select a cost code before running a job.
+    *   `COST_CODES_PATH=config/cost_codes.csv`: The local path to a CSV file containing your cost codes.
+    *   `ENFORCE_COST_CODES=True`: Makes selecting a cost code mandatory before starting a redaction.
+
+Now you have the app installed, please refer to the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for more information on how to use it for basic and advanced redaction.
+
+## For agents (API quickstart)
+
+If you are an LLM/agent interacting with this app over HTTP (e.g. Hugging Face Spaces), **do not guess inputs** from the UI. Use the Gradio schema as the source of truth:
+
+- **Discover schema**: `GET /gradio_api/info`
+- **Upload files**: `POST /gradio_api/upload` (multipart field `files`) → returns server-internal paths like `/tmp/gradio_tmp/...`
+- **Call**: `POST /gradio_api/call/{api_name}` with body `{"data":[...]}` (argument order must match `/gradio_api/info`)
+- **Poll**: `GET /gradio_api/call/{api_name}/{event_id}` until complete
+- **Download outputs**: `GET /gradio_api/file={path}` (note: some deployments return 403 without session cookies)
+
+### Choose the correct route (prefer short `gr.api` endpoints)
+
+Fetch `/gradio_api/info` and then prefer the simplest route that exists:
+
+- **Apply edited review CSV to a PDF**: `/review_apply`
+- **Redact a PDF/image document**: `/doc_redact` — optional `handwrite_signature_checkbox` for AWS Textract (e.g. `Extract handwriting`, `Extract signatures`)
+- **Summarise a PDF**: `/pdf_summarise`
+- **Redact tabular files (CSV/XLSX/Parquet/DOCX)**: `/tabular_redact`
+
+If those endpoints are not present in your deployment, fall back to the long UI-chained routes (`/apply_review_redactions`, `/redact_data`, etc.) and build `data[]` strictly from `/gradio_api/info`.
+
+### Common gotchas
+
+- **Arity errors** (`needed: N, got: M`) mean you called a session-heavy UI handler with the wrong `data[]`. Prefer the short endpoints above.
+- **`handle_file()` gotcha** (for `gradio_client` users): do **not** wrap server-internal upload paths (e.g. `/tmp/gradio_tmp/...`) with `handle_file()`. Pass them as plain strings.
+- **Container-only outputs**: outputs may be written to container paths (e.g. `/home/user/app/output/`). Plan to download via `file=...` or use a mounted output directory in Docker.
+
+### Optional: MCP server
+
+If you want external agents to call this app reliably without re-implementing Gradio upload/call/poll/download details, consider an **MCP server** that wraps the main tasks (`redact_document`, `apply_review_redactions`, `redact_tabular`, `summarise_document`) behind a small tool interface. See the [relevant documentation](https://github.com/seanpedrick-case/doc_redaction/blob/main/mcp_doc_redaction/README.md).
+
+**Use as a library:** After installing from [PyPI](https://pypi.org/project/doc-redaction/) (`pip install doc_redaction`), you can call the same workflows as the Gradio `api_name` routes from Python. See the documentation: [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+    
+To extract text from documents, the 'Local' options are PikePDF for PDFs with selectable text, and OCR with Tesseract. Use AWS Textract to extract more complex elements e.g. handwriting, signatures, or unclear text. PaddleOCR and VLM support is also provided (see the installation instructions below). 
+
+For PII identification, 'Local' (based on spaCy) gives good results if you are looking for common names or terms, or a custom list of terms to redact (see Redaction settings).  AWS Comprehend gives better results at a small cost.
+
+Additional options on the 'Redaction settings' include, the type of information to redact (e.g. people, places), custom terms to include/ exclude from redaction, fuzzy matching, language settings, and whole page redaction. After redaction is complete, you can view and modify suggested redactions on the 'Review redactions' tab to quickly create a final redacted document.
+
+NOTE: The app is not 100% accurate, and it will miss some personal information. It is essential that all outputs are reviewed **by a human** before using the final outputs.

README_PYPI.md

@@ -0,0 +1,351 @@
+# Document redaction (doc_redaction)
+
+<a href="https://pypi.org/project/doc-redaction/" target="_blank"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/doc-redaction"></a> 
+
+Redact personally identifiable information (PII) from documents (PDF, PNG, JPG), Word files (DOCX), or tabular data (XLSX/CSV/Parquet). Please see the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for a full walkthrough of all the features in the app.
+
+---
+
+## 🚀 Quick Start - Installation and first run
+
+Follow these instructions to get the document redaction application running on your local machine.
+
+### 1. Installation
+
+#### Option 1 - Recommended: Install from source repo
+
+Clone the repository and install in editable mode:
+
+```bash
+git clone https://github.com/seanpedrick-case/doc_redaction.git
+cd doc_redaction
+pip install -e .
+```
+
+##### Install extras (Paddle or Transformers/Torch VLM)
+
+To install with PaddleOCR (with a transformers backend as of v2.4.0):
+
+```bash
+pip install -e ".[paddle]"
+```
+
+
+If you want to run VLMs / LLMs with the transformers package:
+
+```bash
+pip install -e ".[vlm]"
+```
+
+
+Note that the versions of both PaddleOCR and Torch installed by default are the CPU-only versions. If you want to install the GPU-enabled version of torch, it is advised to install the following version:
+```bash
+pip install torch==2.10.0 torchvision==0.25.0 --index-url https://download.pytorch.org/whl/cu129
+```
+
+#### Option 2 - Install from PyPI
+
+Create a virtual environment (recommended) and install **doc_redaction**.
+
+```bash
+python -m venv venv
+# Windows:
+.\venv\Scripts\activate
+# macOS/Linux:
+source venv/bin/activate
+```
+
+The package is published on PyPI as **`doc-redaction`** (import name **`doc_redaction`**):
+
+```bash
+pip install doc_redaction
+```
+
+Optional extras (same as in `pyproject.toml`). For installing paddleOCR:
+
+```bash
+pip install "doc_redaction[paddle]"
+```
+
+For running VLMs / LLMs with the transformers package:
+
+```bash
+pip install "doc_redaction[vlm]"
+```
+
+For programmatic use (CLI-first API matching Gradio `api_name` routes), see **[Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html)**. The console script **`cli_redact`** is available after install.
+
+**Web UI from a PyPI install:** You *can* start the Gradio UI after `pip install doc_redaction` by running (note that the prerequisites tesseract and poppler will need to be correctly installed following step 2 below):
+
+```bash
+python -m app
+```
+
+**Important: your working directory matters.** When you run `python -m app`, the app treats your *current folder* as the “app folder”:
+
+- It will look for configuration at `config/app_config.env` *relative to the folder you run it from* (and `python -m doc_redaction.install_deps` will also write `config/app_config.env` there).
+- It may create new folders in that location (for example `config/`, `output/`, `input/`, `logs/`, `usage/`, `feedback/`, and temporary/cache folders depending on your settings).
+- The UI example files and bundled assets are packaged with the PyPI install (they live inside the installed `doc_redaction` package). If you run from a “random” directory after a PyPI install, the app can still locate its packaged examples; your working directory mainly affects where `config/`, `input/`, `output/`, logs, and temp folders are created.
+
+In practice, the **smoothest UI experience** (examples, bundled assets, docs links, predictable relative paths) is still usually via a **repository checkout** or **Docker**, but PyPI install is sufficient to launch the UI as long as you run it from a suitable working folder and have the system dependencies available (or run `python -m doc_redaction.install_deps` first).
+
+#### Option 3 - Docker installation
+
+The doc_redaction Redaction app can be installed by using the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) or Docker compose files ([llama.cpp](https://github.com/ggml-org/llama.cpp), [vLLM](https://docs.vllm.ai/en/stable/)) provided in the repo.
+
+##### With Llama.cpp / vLLM inference server
+
+The project now has Docker and Docker compose files available to pair running the Redaction app with local inference servers powered by [llama.cpp](https://github.com/ggml-org/llama.cpp), or [vLLM](https://docs.vllm.ai/en/stable/). Llama.cpp is more flexible than vLLM for low VRAM systems, as Llama.cpp will offload to cpu/system RAM automatically rather than failing as vLLM tends to do.
+
+For Llama.cpp, you can use the [docker-compose_llama.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_llama.yml) file, and for vLLM, you can use the [docker-compose_vllm.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_vllm.yml) file. To run, Docker / Docker Desktop should be installed, and then you can run the commands suggested in the top of the files to run the servers.
+
+You will need ~40 GB of disk space to run everything depending on the model chosen from the compose file. For the vLLM server, you will need 24 GB VRAM. For the Llama.cpp server, 24 GB VRAM is needed to run at full speed, but the n-gpu-layers and n-cpu-moe parameters in the Docker compose file can be adjusted to fit into your system. I would suggest that 8 GB VRAM is needed as a bare minimum for decent inference speed. See the [Unsloth guide](https://unsloth.ai/docs/models/qwen3.5) for more details on working with GGUF files for Qwen 3.5.
+
+##### Without Llama.cpp / vLLM inference server
+
+If you want a working Docker installation without GPU support, you can install from the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) in the repo. A working example of this, with the CPU version of PaddleOCR, can be found on [Hugging Face](https://huggingface.co/spaces/seanpedrickcase/document_redaction). You can adjust the INSTALL_PADDLEOCR, PADDLE_GPU_ENABLED, INSTALL_VLM, and TORCH_GPU_ENABLED config variables to adjust for PaddleOCR and Transformers packages for local VLM support. Note that GPU-enabled PaddleOCR, and GPU-enabled Transformers/Torch often don't work well together, which is one reason why a Llama.cpp/vLLM inference server Docker installation option is provided below.
+
+The main [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) produces two final images via build targets: **`gradio`** (default web UI, non-root user, named volumes for writable paths) and **`lambda`** (AWS Lambda handler). Build examples:
+
+```bash
+docker build -f Dockerfile --target gradio -t doc-redaction-gradio .
+docker build -f Dockerfile --target lambda -t doc-redaction-lambda .
+```
+
+##### Pi agent (agentic redaction)
+
+The [Pi](https://github.com/earendil-works/pi) orchestration UI uses a separate multi-stage image at [agent-redact/pi-agent/Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/agent-redact/pi-agent/Dockerfile). It shares the same Python 3.12 slim base as the main app; a small Node stage installs the `pi` CLI, which is copied into the runtime image.
+
+| Build target | Typical use |
+|--------------|-------------|
+| **`dev`** | Local development with [docker-compose_llama_agentic.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_llama_agentic.yml) — the repo is bind-mounted; only Pi CLI + Python deps are in the image. |
+| **`runtime`** | [Hugging Face Space](https://huggingface.co/spaces/seanpedrickcase/agentic_document_redaction) and AWS ECS — agent code is baked in; runs as non-root `user` with **named volumes** for workspace, uploads, and session dirs (read-only root filesystem friendly). |
+
+Build from the repository root:
+
+```bash
+docker build -f agent-redact/pi-agent/Dockerfile --target dev -t pi-agent-dev .
+docker build -f agent-redact/pi-agent/Dockerfile --target runtime -t pi-agent-runtime .
+```
+
+For llama.cpp + Pi together, see the compose examples at the top of [docker-compose_llama_agentic.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_llama_agentic.yml). Further detail: [agent-redact/README.md](https://github.com/seanpedrick-case/doc_redaction/blob/main/agent-redact/README.md).
+
+#### Option 4 - Installation on AWS with CDK
+
+The repo contains a [CDK folder](https://github.com/seanpedrick-case/doc_redaction/tree/main/cdk), that contains all the files you need to setup and deploy to an AWS environment with CDK. The installation wizard is [cdk_install.py](https://github.com/seanpedrick-case/doc_redaction/blob/main/cdk/cdk_install.py), which provides a number of options to deploy the Document Redaction App to AWS for demonstration or production. More details on CDK deployment can be found in the [Installation Guide](https://seanpedrick-case.github.io/doc_redaction/src/installation_guide.html).
+
+### 2. Install prerequisites: Tesseract and Poppler
+
+This application relies on two external tools for OCR (Tesseract) and PDF processing (Poppler). Please install them on your system before proceeding.
+
+---
+
+#### Automated dependency setup (recommended)
+
+If you **don’t have admin rights** (or you just want the simplest setup), you can have the project download and configure **Tesseract** and **Poppler** into a local `redaction_deps/` folder inside the doc_redaction folder.
+
+You need the installer script available first, which means either:
+
+- **Repository checkout**: `git clone ...` and run the command from the repo root (recommended for the web UI), or
+- **PyPI install**: `pip install doc_redaction` and run from a writable folder where you want `redaction_deps/` and `config/app_config.env` to be created/updated.
+
+From the repository root (or your chosen working folder) after creating/activating your venv and installing Python requirements:
+
+```bash
+python -m doc_redaction.install_deps
+```
+
+This writes `TESSERACT_FOLDER` / `POPPLER_FOLDER` into `config/app_config.env` so the app can find the binaries without you editing your system PATH.
+
+To just check whether your machine can already see the tools:
+
+```bash
+python -m doc_redaction.install_deps --verify-only
+```
+
+#### **On Windows**
+
+If you don’t use the automated setup above, you can install the dependencies manually by downloading installers and adding the programs to your system's PATH.
+
+1.  **Install Tesseract OCR:**
+    *   Download the installer from the official Tesseract at [UB Mannheim page](https://github.com/UB-Mannheim/tesseract/wiki) (e.g., `tesseract-ocr-w64-setup-v5.X.X...exe`).
+    *   Run the installer.
+    *   **IMPORTANT:** During installation, ensure you select the option to "Add Tesseract to system PATH for all users" or a similar option. This is crucial for the application to find the Tesseract executable.
+
+
+2.  **Install Poppler:**
+    *   Download the latest Poppler binary for Windows. A common source is the [Poppler for Windows](https://github.com/oschwartz10612/poppler-windows) GitHub releases page. Download the `.zip` file (e.g., `poppler-25.07.0-win.zip`).
+    *   Extract the contents of the zip file to a permanent location on your computer, for example, `C:\Program Files\poppler\`.
+    *   You must add the `bin` folder from your Poppler installation to your system's PATH environment variable.
+        *   Search for "Edit the system environment variables" in the Windows Start Menu and open it.
+        *   Click the "Environment Variables..." button.
+        *   In the "System variables" section, find and select the `Path` variable, then click "Edit...".
+        *   Click "New" and add the full path to the `bin` directory inside your Poppler folder (e.g., `C:\Program Files\poppler\poppler-24.02.0\bin`).
+        *   Click OK on all windows to save the changes.
+
+    To verify, open a new Command Prompt and run `tesseract --version` and `pdftoppm -v`. If they both return version information, you have successfully installed the prerequisites.
+---
+
+#### **On Linux (Debian/Ubuntu)**
+
+Open your terminal and run the following command to install Tesseract and Poppler:
+
+```bash
+sudo apt-get update && sudo apt-get install -y tesseract-ocr poppler-utils
+```
+
+#### **On Linux (Fedora/CentOS/RHEL)**
+
+Open your terminal and use the `dnf` or `yum` package manager:
+
+```bash
+sudo dnf install -y tesseract poppler-utils
+```
+---
+
+### 3. Run the Application
+
+With all dependencies installed, you can now start the Gradio application GUI. For a guide on how to use this, please go [here](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html).
+
+```bash
+python app.py
+```
+
+After running the command, the application will start, and you will see a local URL in your terminal (usually `http://127.0.0.1:7860`).
+
+Open this URL in your web browser to use the document redaction tool
+
+#### Command line interface
+
+For example CLI commands, please refer to [this guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html#command-line-interface-cli) or the examples in [cli_redact.py](https://github.com/seanpedrick-case/doc_redaction/blob/main/cli_redact.py#L321)
+
+If you installed from **PyPI**, use the installed console script:
+
+```bash
+cli_redact --help
+```
+
+From a **repository checkout**, you can also run:
+
+```bash
+python cli_redact.py --help
+```
+
+#### Python package commands
+
+For Python examples in using the Python package, please see [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+
+---
+
+
+### 4. ⚙️ Configuration (Optional)
+
+You can customise the application's behavior by creating a configuration file. This allows you to change settings without modifying the source code, such as enabling AWS features, changing logging behavior, or pointing to local Tesseract/Poppler installations. A full overview of all the potential settings you can modify in the app_config.env file can be seen in tools/config.py, with explanation on the documentation website for [the github repo](https://seanpedrick-case.github.io/doc_redaction/)
+
+To get started:
+1.  Copy `config/app_config.env.example` to `config/app_config.env`.
+2.  Modify the values in `config/app_config.env` to suit your needs. The application will automatically load these settings on startup.
+
+If you do not create this file, the application will run with default settings.
+
+#### Configuration Breakdown
+
+Here is an overview of the most important settings, separated by whether they are for local use or require AWS.
+
+---
+
+#### **Local & General Settings (No AWS Required)**
+
+These settings are useful for all users, regardless of whether you are using AWS.
+
+*   `TESSERACT_FOLDER` / `POPPLER_FOLDER`
+    *   Use these if you installed Tesseract or Poppler to a custom location on **Windows** and did not add them to the system PATH.
+    *   Provide the path to the respective installation folders (for Poppler, point to the `bin` sub-directory).
+    *   **Examples:** `POPPLER_FOLDER=C:/Program Files/poppler-24.02.0/bin/` `TESSERACT_FOLDER=tesseract/`
+
+*   `SHOW_LANGUAGE_SELECTION=True`
+    *   Set to `True` to display a language selection dropdown in the UI for OCR processing.
+
+*   `DEFAULT_LOCAL_OCR_MODEL=tesseract`"
+    *   Choose the backend for local OCR. Options are `tesseract`, `paddle`, or `hybrid`. "Tesseract" is the default, and is recommended. "hybrid-paddle" is a combination of the two - first pass through the redactions will be done with Tesseract, and then a second pass will be done with PaddleOCR on words with low confidence. "paddle" will only return whole line text extraction, and so will only work for OCR, not redaction. 
+
+*   `SESSION_OUTPUT_FOLDER=False`
+    *   If `True`, redacted files will be saved in unique subfolders within the `output/` directory for each session.
+
+*   `DISPLAY_FILE_NAMES_IN_LOGS=False`
+    *   For privacy, file names are not recorded in usage logs by default. Set to `True` to include them.
+
+---
+
+#### **AWS-Specific Settings**
+
+These settings are only relevant if you intend to use AWS services like Textract for OCR and Comprehend for PII detection.
+
+*   `RUN_AWS_FUNCTIONS=True`
+    *   **This is the master switch.** You must set this to `True` to enable any AWS functionality. If it is `False`, all other AWS settings will be ignored.
+
+*   **UI Options:**
+    *   `SHOW_AWS_TEXT_EXTRACTION_OPTIONS=True`: Adds "AWS Textract" as an option in the text extraction dropdown.
+    *   `SHOW_AWS_PII_DETECTION_OPTIONS=True`: Adds "AWS Comprehend" as an option in the PII detection dropdown.
+
+*   **Core AWS Configuration:**
+    *   `AWS_REGION=example-region`: Set your AWS region (e.g., `us-east-1`).
+    *   `DOCUMENT_REDACTION_BUCKET=example-bucket`: The name of the S3 bucket the application will use for temporary file storage and processing.
+
+*   **AWS Logging:**
+    *   `SAVE_LOGS_TO_DYNAMODB=True`: If enabled, usage and feedback logs will be saved to DynamoDB tables.
+    *   `ACCESS_LOG_DYNAMODB_TABLE_NAME`, `USAGE_LOG_DYNAMODB_TABLE_NAME`, etc.: Specify the names of your DynamoDB tables for logging.
+
+*   **Advanced AWS Textract Features:**
+    *   `SHOW_WHOLE_DOCUMENT_TEXTRACT_CALL_OPTIONS=True`: Enables UI components for large-scale, asynchronous document processing via Textract.
+    *   `TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET=example-bucket-output`: A separate S3 bucket for the final output of asynchronous Textract jobs.
+    *   `LOAD_PREVIOUS_TEXTRACT_JOBS_S3=True`: If enabled, the app will try to load the status of previously submitted asynchronous jobs from S3.
+
+*   **Cost Tracking (for internal accounting):**
+    *   `SHOW_COSTS=True`: Displays an estimated cost for AWS operations. Can be enabled even if AWS functions are off.
+    *   `GET_COST_CODES=True`: Enables a dropdown for users to select a cost code before running a job.
+    *   `COST_CODES_PATH=config/cost_codes.csv`: The local path to a CSV file containing your cost codes.
+    *   `ENFORCE_COST_CODES=True`: Makes selecting a cost code mandatory before starting a redaction.
+
+Now you have the app installed, please refer to the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for more information on how to use it for basic and advanced redaction.
+
+## For agents (API quickstart)
+
+If you are an LLM/agent interacting with this app over HTTP (e.g. Hugging Face Spaces), **do not guess inputs** from the UI. Use the Gradio schema as the source of truth:
+
+- **Discover schema**: `GET /gradio_api/info`
+- **Upload files**: `POST /gradio_api/upload` (multipart field `files`) → returns server-internal paths like `/tmp/gradio_tmp/...`
+- **Call**: `POST /gradio_api/call/{api_name}` with body `{"data":[...]}` (argument order must match `/gradio_api/info`)
+- **Poll**: `GET /gradio_api/call/{api_name}/{event_id}` until complete
+- **Download outputs**: `GET /gradio_api/file={path}` (note: some deployments return 403 without session cookies)
+
+### Choose the correct route (prefer short `gr.api` endpoints)
+
+Fetch `/gradio_api/info` and then prefer the simplest route that exists:
+
+- **Apply edited review CSV to a PDF**: `/review_apply`
+- **Redact a PDF/image document**: `/doc_redact` — optional `handwrite_signature_checkbox` for AWS Textract (e.g. `Extract handwriting`, `Extract signatures`)
+- **Summarise a PDF**: `/pdf_summarise`
+- **Redact tabular files (CSV/XLSX/Parquet/DOCX)**: `/tabular_redact`
+
+If those endpoints are not present in your deployment, fall back to the long UI-chained routes (`/apply_review_redactions`, `/redact_data`, etc.) and build `data[]` strictly from `/gradio_api/info`.
+
+### Common gotchas
+
+- **Arity errors** (`needed: N, got: M`) mean you called a session-heavy UI handler with the wrong `data[]`. Prefer the short endpoints above.
+- **`handle_file()` gotcha** (for `gradio_client` users): do **not** wrap server-internal upload paths (e.g. `/tmp/gradio_tmp/...`) with `handle_file()`. Pass them as plain strings.
+- **Container-only outputs**: outputs may be written to container paths (e.g. `/home/user/app/output/`). Plan to download via `file=...` or use a mounted output directory in Docker.
+
+### Optional: MCP server
+
+If you want external agents to call this app reliably without re-implementing Gradio upload/call/poll/download details, consider an **MCP server** that wraps the main tasks (`redact_document`, `apply_review_redactions`, `redact_tabular`, `summarise_document`) behind a small tool interface. See the [relevant documentation](https://github.com/seanpedrick-case/doc_redaction/blob/main/mcp_doc_redaction/README.md).
+
+**Use as a library:** After installing from [PyPI](https://pypi.org/project/doc-redaction/) (`pip install doc_redaction`), you can call the same workflows as the Gradio `api_name` routes from Python. See the documentation: [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+    
+To extract text from documents, the 'Local' options are PikePDF for PDFs with selectable text, and OCR with Tesseract. Use AWS Textract to extract more complex elements e.g. handwriting, signatures, or unclear text. PaddleOCR and VLM support is also provided (see the installation instructions below). 
+
+For PII identification, 'Local' (based on spaCy) gives good results if you are looking for common names or terms, or a custom list of terms to redact (see Redaction settings).  AWS Comprehend gives better results at a small cost.
+
+Additional options on the 'Redaction settings' include, the type of information to redact (e.g. people, places), custom terms to include/ exclude from redaction, fuzzy matching, language settings, and whole page redaction. After redaction is complete, you can view and modify suggested redactions on the 'Review redactions' tab to quickly create a final redacted document.
+
+NOTE: The app is not 100% accurate, and it will miss some personal information. It is essential that all outputs are reviewed **by a human** before using the final outputs.

agent-redact/README.md

@@ -0,0 +1,33 @@
+# Pi-based agentic document redaction: local Docker orchestration and Hugging Face Space packaging.
+
+Supports three orchestration backends via `AGENT_ORCHESTRATOR` (`pi` default, `langgraph`, `agentcore`). See [`pi/agent/README.md`](pi/agent/README.md).
+
+| Path | Purpose |
+|------|---------|
+| [`shared/`](shared/) | Gradio UI, runtime factory, session workspace, remote redaction helpers |
+| [`pi/`](pi/) | Pi RPC client, Pi config generation, skills sync, `agent/*.json` templates |
+| [`redaction_langgraph/`](redaction_langgraph/) | LangGraph ReAct agent (curated tools, no shell) |
+| [`agentcore/`](agentcore/) | Bedrock AgentCore runtime + Gradio AgentCore clients + **[install guide](agentcore/README.md)** |
+| [`eval/`](eval/) | Arize AX / Phoenix tracing for LangGraph |
+| [`pi-agent/`](pi-agent/) | Docker image (`dev` + `runtime` targets), sync script, and manifest |
+| [`requirements_agent.txt`](requirements_agent.txt) | Python deps for agent-redact Docker images |
+
+Per-user output isolation uses Gradio `session_hash` subfolders under `AGENT_WORKSPACE_DIR` (see `agent-redact/shared/session_workspace.py`). Enabled by default locally and on HF Spaces. Set `AGENT_SESSION_WORKSPACE=false` only if you want one shared workspace tree for all sessions.
+
+## Local Docker
+
+Use the `pi-agent` service in [`docker-compose_llama_agentic.yml`](../docker-compose_llama_agentic.yml) (profile `27b_36`). See [`pi/agent/README.md`](pi/agent/README.md).
+
+## Hugging Face Space
+
+Build from repo root:
+
+```bash
+# Production (HF Space / ECS)
+docker build -f agent-redact/pi-agent/Dockerfile --target runtime .
+
+# Local compose (bind-mounted repo)
+docker build -f agent-redact/pi-agent/Dockerfile --target dev .
+```
+
+Sync to Space on pushes to `dev` via [`.github/workflows/sync-pi-agent-space.yml`](../.github/workflows/sync-pi-agent-space.yml).

agent-redact/agentcore/Dockerfile.runtime

@@ -0,0 +1,38 @@
+# syntax=docker/dockerfile:1
+# Bedrock AgentCore Runtime image (linux/arm64). Build from monorepo root:
+#   docker build --platform linux/arm64 -f agent-redact/agentcore/Dockerfile.runtime .
+#
+# AgentCore requires arm64, port 8080, POST /invocations and GET /ping
+# (provided by BedrockAgentCoreApp in entrypoint.py).
+
+FROM --platform=linux/arm64 public.ecr.aws/docker/library/python:3.12.13-slim-trixie
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV APP_TYPE=agent
+ENV APP_HOME=/app
+ENV PYTHONPATH=${APP_HOME}:${APP_HOME}/agent-redact:${APP_HOME}/agent-redact/shared:${APP_HOME}/agent-redact/pi:${APP_HOME}/agent-redact/agentcore
+ENV AGENT_WORKSPACE_DIR=/tmp/agentcore-workspace
+ENV AWS_REGION=eu-west-2
+ENV AWS_DEFAULT_REGION=${AWS_REGION}
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+WORKDIR ${APP_HOME}
+
+COPY agent-redact/requirements_agent.txt /tmp/requirements_agent.txt
+RUN pip install --no-cache-dir -r /tmp/requirements_agent.txt \
+    && rm /tmp/requirements_agent.txt
+
+COPY agent-redact/ ${APP_HOME}/agent-redact/
+COPY config/agent.env.example ${APP_HOME}/config/agent.env.example
+
+RUN mkdir -p /tmp/agentcore-workspace \
+    && chmod 1777 /tmp/agentcore-workspace
+
+EXPOSE 8080
+
+CMD ["python", "agent-redact/agentcore/entrypoint.py"]

agent-redact/agentcore/README.md

@@ -0,0 +1,470 @@
+# Bedrock AgentCore install guide
+
+This folder contains the **AgentCore Runtime entrypoint** ([`entrypoint.py`](entrypoint.py)) — a LangGraph redaction agent wrapped in `BedrockAgentCoreApp`.
+
+The **Gradio agent UI** (Pi Express / legacy ECS from [`cdk/cdk_install.py`](../../cdk/cdk_install.py)) stays the user-facing app. When `AGENT_ORCHESTRATOR=agentcore`, that UI proxies prompts to a **separately deployed** AgentCore Runtime via `AGENTCORE_RUNTIME_URL`.
+
+You do **not** define `AGENTCORE_RUNTIME_URL` manually in the AWS console beforehand. It is the **invoke endpoint AWS returns after you deploy** an AgentCore Runtime.
+
+## Architecture (two parts)
+
+| Component | Deployed by | Role |
+|-----------|-------------|------|
+| **AgentCore Runtime** | [AgentCore CLI](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-get-started-cli.html) (`agentcore deploy`) | Runs the LangGraph agent (`entrypoint.py`) |
+| **Gradio agent UI** | doc_redaction CDK / `cdk_install.py` (Pi Express or legacy) | Browser UI; streams to AgentCore when `AGENT_ORCHESTRATOR=agentcore` or `agentcore-harness` |
+
+The main **doc_redaction** app (OCR, PII, `/doc_redact`, `/review_apply`) is unchanged.
+
+- **Pi / LangGraph in the Pi Express container** can call the main app over ECS Service Connect (`http://redaction:7860`).
+- **Bedrock AgentCore Runtime** (separate AWS service) uses the **main Express public HTTPS URL** (`ExpressServiceEndpoint` stack output). CDK sets that on Pi Express when `ENABLE_AGENTCORE_RUNTIME=True`; Gradio passes it to AgentCore on each invoke via `runtime_config`.
+
+## Runtime vs Harness
+
+| | **AgentCore Runtime** (`agentcore`) | **AgentCore Harness** (`agentcore-harness`) |
+|--|-------------------------------------|---------------------------------------------|
+| AWS resource | `arn:...:runtime/...` | `arn:...:harness/...` |
+| Config | `AGENTCORE_RUNTIME_URL` (HTTP base, no `/invocations`) | `AGENTCORE_HARNESS_ARN` |
+| Invoke | `InvokeAgentRuntime` / HTTP SSE | `InvokeHarness` (boto3 stream) |
+| Agent code | Your LangGraph bundle (`package_runtime.py`) | AWS-managed Strands loop; tools/skills in console |
+| Redaction prompt | Tool orchestrator (curated LangGraph tools) | Pi-like partnership prompt (skills + shell) |
+| File upload from Gradio | Base64 `workspace_files` in invoke payload | S3 presigned URL prefix (`AGENTCORE_HARNESS_S3_INPUT_PREFIX`) |
+
+There is **no HTTP invocation URL** for a Harness — configure the **ARN** and call the SDK. The console does not show a Runtime-style URL for Harness resources.
+
+## What `AGENTCORE_RUNTIME_URL` is
+
+In this repo, set the **base URL only** (no trailing slash, **no** `/invocations` suffix). The Gradio client in [`agentcore_runtime.py`](agentcore_runtime.py) calls:
+
+```text
+{AGENTCORE_RUNTIME_URL}/invocations
+```
+
+Example base (region and ARN are yours):
+
+```text
+https://bedrock-agentcore.eu-west-2.amazonaws.com/runtimes/arn%3Aaws%3Abedrock-agentcore%3Aeu-west-2%3A123456789012%3Aruntime%2FRedactionAgent
+```
+
+Full invoke URLs often include `?qualifier=DEFAULT`; this project appends `/invocations` to the base you configure.
+
+## Prerequisites
+
+- AWS account with credentials configured (`aws sts get-caller-identity`)
+- [Node.js 20+](https://nodejs.org/) for the AgentCore CLI
+- Python 3.10+
+- [AWS CDK bootstrapped](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html) in the target account/region (`cdk bootstrap`)
+- Bedrock model access enabled if the agent uses Bedrock models
+- IAM permissions for AgentCore deploy and `bedrock-agentcore:InvokeAgentRuntime` (see [Use the AgentCore CLI](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-get-started-cli.html))
+
+Install the CLI:
+
+```bash
+npm install -g @aws/agentcore
+agentcore --help
+```
+
+If you previously installed the old Python “starter toolkit” CLI, uninstall it first — both use the `agentcore` command name (`pip uninstall bedrock-agentcore-starter-toolkit` if applicable).
+
+## Step 1 — Deploy the AgentCore runtime
+
+### Option A: New AgentCore project (recommended first time)
+
+The AgentCore CLI creates a **new project folder** next to where you run the command. It does **not** add `app/RedactionAgent/` inside `doc_redaction/agent-redact/agentcore/`.
+
+**Run `create` from a parent directory** (repo root, `agent-redact/`, or `~/projects`):
+
+```bash
+# Code-based LangGraph agent (do NOT use --defaults — that creates a harness-only project)
+agentcore create \
+  --name RedactionAgent \
+  --framework LangChain_LangGraph \
+  --model-provider Bedrock \
+  --memory none
+
+# Or interactive (choose "Agent", then LangChain/LangGraph, Bedrock, memory none):
+# agentcore create
+
+cd RedactionAgent
+ls app/RedactionAgent
+```
+
+**Why `RedactionAgent/` might not appear**
+
+| What you ran | What happened |
+|--------------|----------------|
+| `... --framework LangChain_LangGraph --defaults` | **Invalid combo.** `--defaults` means “create a **harness** project”, not “fill in missing flags”. The CLI exits after: *Use --no-agent for project-only, or provide all: --framework, --model-provider, --memory* — **no folder is created.** |
+| Missing `--model-provider` / `--memory` | Same message; add both flags (see command above). |
+
+Non-interactive **code agent** requires all three: `--framework`, `--model-provider`, `--memory`. See `agentcore create --help`.
+
+Optional: `--output-dir ..` to create the project next to `agent-redact/` instead of inside it.
+
+Expected layout after `create` (project name = `--name` value):
+
+```text
+RedactionAgent/
+├── agentcore/
+│   ├── agentcore.json      # created by CLI — agent/runtime config
+│   ├── aws-targets.json    # created by CLI — edit account/region (see below)
+│   └── cdk/                # auto-managed CDK for deploy
+└── app/
+    └── RedactionAgent/     # same name as --name
+        ├── main.py         # generated entrypoint — you edit this
+        └── pyproject.toml
+```
+
+If you do **not** see `RedactionAgent/` (or `app/<name>/` after `cd`):
+
+| Symptom | Likely cause |
+|---------|----------------|
+| No new folder at all | Incomplete non-interactive flags, or used `--defaults` with `--framework` — use full command above |
+| Only `agentcore/` files where you ran the command | You may have run `create` inside an existing AgentCore tree; run it from a clean parent directory instead |
+| No `app/` subdirectory | Interactive wizard chose **Harness** or **Skip** — run `agentcore add agent` or recreate with `--framework LangChain_LangGraph` |
+| Looking for `app/RedactionAgent` inside `agent-redact/agentcore/` | Wrong place — that folder only holds this repo’s reference [`entrypoint.py`](entrypoint.py), not the CLI scaffold |
+
+Official reference: [AgentCore get started](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/agentcore-get-started-cli.html).
+
+#### Using this repo’s `entrypoint.py`
+
+Do **not** only rename `entrypoint.py` to `main.py`. The CLI already generates `app/RedactionAgent/main.py` with starter LangGraph code.
+
+1. Open the generated `app/RedactionAgent/main.py`.
+2. Replace its handler logic with the code from [`entrypoint.py`](entrypoint.py) in this repo (the `@app.entrypoint` async `handler` and `BedrockAgentCoreApp` setup).
+3. Add dependencies to `app/RedactionAgent/pyproject.toml` (e.g. `bedrock-agentcore`, `langgraph`, `langchain-*`) matching [`requirements_agent.txt`](../requirements_agent.txt).
+4. **Package monorepo code for deploy** — the generated project does not automatically include `redaction_langgraph/`, `tools/`, or `skills/`. Typical approaches:
+   - **Container build** (`--build Container`): copy or mount the needed paths from `doc_redaction` in the Dockerfile the CLI scaffolds; or
+   - **Vendor** `agent-redact/redaction_langgraph/` and required `tools/` modules into `app/RedactionAgent/` before deploy.
+
+[`entrypoint.py`](entrypoint.py) is a **reference implementation** for this monorepo; AgentCore deploy packages whatever is under `app/<AgentName>/`, not the whole `doc_redaction` tree unless you wire that in.
+
+#### `aws-targets.json` (created for you — edit, don’t invent)
+
+You do **not** need to create this file manually. `agentcore create` writes `agentcore/aws-targets.json` inside the new project.
+
+Edit it so `account` and `region` match where you will deploy (often `us-west-2` or `eu-west-2` for AgentCore):
+
+```json
+[
+  {
+    "name": "default",
+    "description": "doc_redaction AgentCore deploy",
+    "account": "123456789012",
+    "region": "eu-west-2"
+  }
+]
+```
+
+Get your account ID: `aws sts get-caller-identity --query Account --output text`.
+
+Schema reference: [agentcore-cli configuration](https://github.com/aws/agentcore-cli/blob/main/docs/configuration.md).
+
+Then deploy:
+
+```bash
+# still inside RedactionAgent/
+agentcore dev          # optional: local test at http://localhost:8080/invocations
+agentcore deploy
+```
+
+### Option B: Package and deploy the doc_redaction agent (recommended)
+
+Use [`package_runtime.py`](package_runtime.py) to sync `redaction_langgraph`, Pi helpers, session memory, and `main.py` into your AgentCore app folder — then deploy.
+
+**Prerequisites:** `agentcore create` project at `agent-redact/RedactionAgent/` (or pass `--target`).
+
+From the **doc_redaction repo root**:
+
+```powershell
+# Preview
+python agent-redact/agentcore/package_runtime.py --dry-run
+
+# Package into agent-redact/RedactionAgent/app/RedactionAgent/
+python agent-redact/agentcore/package_runtime.py
+
+# Package + deploy (set UV_LINK_MODE on Windows / OneDrive)
+$env:UV_LINK_MODE = "copy"
+python agent-redact/agentcore/package_runtime.py --deploy
+```
+
+Or manually after packaging:
+
+```powershell
+cd agent-redact\RedactionAgent
+agentcore validate
+agentcore deploy
+```
+
+**What the script copies**
+
+| Source | Destination in `app/RedactionAgent/` |
+|--------|--------------------------------------|
+| `redaction_langgraph/` | `redaction_langgraph/` |
+| `pi/bootstrap_pi_config.py`, `remote_redaction.py` | `pi/` |
+| `agentcore/bundle_support/session_workspace.py` | `pi/session_workspace.py` (no Gradio dep) |
+| `agentcore/invoke_agent.py`, `session_store.py` | app root |
+| Generated `main.py` | replaces template `main.py` |
+| Runtime deps | merged into `pyproject.toml` |
+| `agentcore.env.example` | env vars to set on the **AWS runtime** |
+
+**After deploy — runtime environment (AWS)**
+
+Bedrock model settings (`AGENT_DEFAULT_PROVIDER`, `AWS_REGION`, …) belong in `agentcore.env` on the runtime bundle.
+
+**`DOC_REDACTION_GRADIO_URL`:** the Gradio Pi UI sends this on **every invoke** in `runtime_config`, taken from your local `config/agent.env`. That overrides any URL baked into `agentcore.env` (for example an old HF Space default). You should see `Redaction backend for this turn: …` in the activity log with the same URL as the session info panel.
+
+For AWS CDK + AgentCore, `DOC_REDACTION_GRADIO_URL` is the **main app HTTPS URL** (`CloudFrontDistributionURL` when `USE_CLOUDFRONT=True`, otherwise `ExpressServiceEndpoint` / `AgenticDocRedactionBackendUrl` stack output). Service Connect (`http://redaction:7860`) is only for in-container `pi` / `langgraph` orchestrators. For local Docker dev, set `DOC_REDACTION_GRADIO_URL=http://host.docker.internal:7861` in `agent.env`.
+
+```bash
+AGENT_DEFAULT_PROVIDER=amazon-bedrock
+AGENT_DEFAULT_MODEL=anthropic.claude-sonnet-4-6
+AWS_REGION=eu-west-2
+AGENT_WORKSPACE_DIR=/tmp/agentcore-workspace
+# Optional fallback if Gradio does not send runtime_config:
+# DOC_REDACTION_GRADIO_URL=https://<ExpressServiceEndpoint>  # CDK + AgentCore
+```
+
+**Session / follow-up chat:** [`session_store.py`](session_store.py) keeps conversation history per `session_hash` inside the running runtime process. Gradio passes the same `session_hash` for follow-ups. History is lost on cold start until [AgentCore Memory](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/) is configured.
+
+**Workspace file sync (Gradio ↔ AgentCore):** AgentCore runs on AWS with its own filesystem (`AGENT_WORKSPACE_DIR`). When you **Start redaction task** from the Gradio UI, the uploaded PDF is base64-encoded in the invoke payload (`workspace_files`) and written into the remote session workspace before the LangGraph agent runs. After the turn completes, artifacts under `redact/` are streamed back as `workspace_file` events and saved into the local Gradio session workspace so the **Outputs** panel can refresh. Default per-file limit: 8 MB (`AGENTCORE_MAX_UPLOAD_BYTES`). Re-deploy the runtime after upgrading `invoke_agent.py` / `workspace_sync.py`.
+
+#### Alternative: Container build
+
+For a full monorepo checkout in the image, switch the runtime in `agentcore/agentcore.json` to `"build": "Container"`, add a `Dockerfile` under `app/RedactionAgent/` that `COPY`s the repo and sets `CMD` to run the entrypoint, then `agentcore deploy`.
+
+## Step 2 — Get the runtime URL
+
+After a successful deploy:
+
+```bash
+agentcore status
+# or non-interactive (PowerShell-friendly):
+agentcore status --json
+```
+
+- **`agentcore status`** — runtime ARN, `invocationUrl`, deployment state, log hints (this is what you want for the HTTP endpoint)
+- **`agentcore fetch access`** — only for **gateways** or agents using **CUSTOM_JWT** inbound auth (fetches bearer token guidance). It does **not** apply to the default AWS_IAM runtime agent; running bare `agentcore fetch` only prints subcommand help.
+
+Example (your deployed runtime):
+
+```text
+AGENTCORE_RUNTIME_URL=https://bedrock-agentcore.eu-west-2.amazonaws.com/runtimes/arn%3Aaws%3Abedrock-agentcore%3Aeu-west-2%3A404053085091%3Aruntime%2FRedactionAgent_RedactionAgent-ye5Jfw7gKj
+```
+
+Use the `invocationUrl` from `agentcore status --json` but **drop the trailing `/invocations`** — the Gradio client appends that path. Auth is **SigV4 (AWS IAM)**, not a static API key, unless you later configure CUSTOM_JWT on the runtime.
+
+You can also find the runtime in the AWS console under **Amazon Bedrock → AgentCore** (runtime resources created by deploy).
+
+### Build the URL from the runtime ARN
+
+If you only have the ARN, the HTTP base is typically:
+
+```text
+https://bedrock-agentcore.<region>.amazonaws.com/runtimes/<URL-encoded-ARN>
+```
+
+URL-encode the ARN (`:` → `%3A`, `/` → `%2F`). See the [HTTP protocol contract](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-http-protocol-contract.html).
+
+### Test invoke (CLI)
+
+```bash
+agentcore invoke --runtime RedactionAgent "Run Pass 1 redaction on the uploaded PDF"
+agentcore invoke --runtime RedactionAgent "Hello" --stream
+```
+
+Programmatic invoke uses the AWS SDK `InvokeAgentRuntime` API with the runtime ARN ([AWS docs](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-get-started-cli.html)).
+
+## Step 3 — Wire the Gradio agent UI
+
+### Demonstration (CDK) install
+
+The [CDK installer](../../cdk/cdk_install.py) demo profile (`--profile demo --enable-pi`) defaults to **AgentCore orchestration** for the Express agent Gradio UI. The Pi coding-agent CLI remains in the container image but is unused when `AGENT_ORCHESTRATOR=agentcore`.
+
+#### Option C — CDK-native runtime from ECR (recommended for demo)
+
+The installer can build an **ARM64** runtime image from this monorepo (CodeBuild → ECR) and create a **Bedrock AgentCore Runtime** via CDK (`aws_bedrockagentcore.CfnRuntime`) — no local `agentcore deploy` CLI required.
+
+**Phase 1 — infra + runtime image**
+
+```powershell
+python cdk/cdk_install.py --profile demo --enable-agentic `
+  --enable-agentcore-cdk-deploy --yes
+```
+
+This sets `AGENTCORE_CDK_DEPLOY=True`, creates an ARM CodeBuild project and ECR repo (`agent-redact/agentcore/Dockerfile.runtime`), and runs `post_cdk_build_quickstart.py` to push the image.
+
+**Phase 2 — create runtime + wire URL (no stack update)**
+
+Phase 1 runs `post_cdk_build_quickstart.py`, which triggers the AgentCore CodeBuild and, if the image is ready before quickstart finishes, completes phase 2 automatically. If the image is **not** ready yet (or CodeBuild failed and you pushed the image manually), complete phase 2 once `agent-redact/agentcore/Dockerfile.runtime` has produced `…-agentcore-runtime:latest` in ECR:
+
+```powershell
+python cdk/cdk_install.py --complete-agentcore
+```
+
+`--complete-agentcore` uses the existing `cdk/config/cdk_config.env`, skips the wizard, and **does not run `cdk deploy`**. It calls the `bedrock-agentcore-control` API directly (boto3) to create the runtime from the ECR image + the phase-1 execution role (`AgentCoreRuntimeExecutionRole`, mirroring the CDK `CfnRuntime` parameters), then derives `AGENTCORE_RUNTIME_URL`, patches `config/agent.env` / `cdk/config/cdk_config.env`, re-uploads to S3, and recycles the agent Express service. It is idempotent — reuses an existing runtime of the same name.
+
+> **Why not a second `cdk deploy`?** `cdk_install.py` performs **initial deploys only** and aborts rather than update an existing stack. This is deliberate: `app.py` re-runs the precheck on every `cdk` invocation, and a second deploy would find the stack's now-existing roles/buckets/pool, flip them from *managed* to *imported*, and CloudFormation would then **delete** them (removal policy `DESTROY` on the demo profile). So phase 2 creates the runtime out-of-band via the API instead — no stack mutation, no footgun. If you later tear down the stack, delete the AgentCore runtime separately (it is not tracked by CloudFormation).
+
+Config keys (also in `cdk/config/cdk_config.env`):
+
+| Variable | Role |
+|----------|------|
+| `AGENTCORE_CDK_DEPLOY` | Phase 1: CodeBuild + ECR + execution IAM role |
+| `ENABLE_AGENTCORE_CDK_RUNTIME` | Phase 2: create `AWS::BedrockAgentCore::Runtime` |
+| `ECR_AGENTCORE_REPO_NAME` | ECR repository for the runtime image |
+| `CODEBUILD_AGENTCORE_PROJECT_NAME` | ARM64 CodeBuild project |
+| `AGENTCORE_RUNTIME_NAME` | Runtime resource name (default `{CDK_PREFIX}RedactionAgent`) |
+
+The runtime image is built from the **doc_redaction monorepo** (`GITHUB_REPO_*`); it copies `agent-redact/`, `redaction_langgraph/`, and `pi/` helpers — no separate GitHub repo required.
+
+#### Option A/B — manual CLI deploy (alternative)
+
+Deploy is **two-phase** when using the AgentCore CLI — the runtime URL does not exist until after `agentcore deploy`:
+
+**Phase 1 — AgentCore runtime (before or after CDK; URL required before the agent UI works)**
+
+```powershell
+# From doc_redaction repo root (after agentcore create — see Option A/B above)
+python agent-redact/agentcore/package_runtime.py `
+  --target C:\path\to\RedactionAgent\app\RedactionAgent
+
+cd C:\path\to\RedactionAgent
+$env:UV_LINK_MODE = "copy"
+agentcore deploy
+agentcore status   # copy invocationUrl (base only, no /invocations)
+```
+
+Set runtime env on AWS so tools reach doc_redaction. For CDK + AgentCore use the **main Express HTTPS URL** (`ExpressServiceEndpoint`); `agentcore.env` is only a fallback — Gradio overrides via `runtime_config` each invoke.
+
+**Phase 2 — CDK demo stack**
+
+```powershell
+# With runtime URL (recommended)
+python cdk/cdk_install.py --profile demo --enable-pi `
+  --agentcore-runtime-url "https://bedrock-agentcore....amazonaws.com/runtimes/..." --yes
+
+# Or config-only first; add URL after runtime deploy
+python cdk/cdk_install.py --profile demo --enable-pi --yes --config-only
+# then re-run with --agentcore-runtime-url or edit config/agent.env
+```
+
+The installer adds `policies/pi_agentcore_invoke_policy.json` to the ECS task role so the Gradio container can call `InvokeAgentRuntime` (SigV4, no `AGENTCORE_API_KEY` required on ECS).
+
+**Order B (infra first):** deploy CDK with `--agent-orchestrator pi` or defer the URL in interactive mode, deploy the runtime, then `cdk_install.py --config-only --agentcore-runtime-url <URL>` and restart the Pi Express service.
+
+**Fallback orchestrators:** `--agent-orchestrator pi` or `langgraph` runs orchestration inside the Express container (no separate AgentCore runtime).
+
+---
+
+Set in [`config/agent.env`](../../config/agent.env.example) (or via the CDK installer):
+
+```bash
+AGENT_ORCHESTRATOR=agentcore
+AGENTCORE_RUNTIME_URL=https://bedrock-agentcore.eu-west-2.amazonaws.com/runtimes/...
+# Optional if the runtime uses bearer/OAuth inbound auth:
+AGENTCORE_API_KEY=your-bearer-token
+```
+
+**Harness orchestrator** (console-created Harness, Pi-like skills/shell — not the LangGraph bundle):
+
+```bash
+AGENT_ORCHESTRATOR=agentcore-harness
+AGENTCORE_HARNESS_ARN=arn:aws:bedrock-agentcore:eu-west-2:...:harness/YourHarness-xyz
+# Optional endpoint name (default DEFAULT):
+# AGENTCORE_HARNESS_ENDPOINT=DEFAULT
+# S3 file bridge for Start redaction task (upload PDF + presigned URL in prompt):
+# AGENTCORE_HARNESS_S3_INPUT_PREFIX=s3://your-bucket/harness-inputs/
+# AGENTCORE_HARNESS_S3_MOUNT_PATH=/tmp/workspace
+RUN_AWS_FUNCTIONS=True
+```
+
+Client: [`agentcore_harness_runtime.py`](agentcore_harness_runtime.py). Requires a recent `boto3` with `invoke_harness`.
+
+### CDK installer (non-interactive)
+
+```bash
+python cdk/cdk_install.py --profile demo --enable-pi \
+  --agent-orchestrator agentcore \
+  --agentcore-runtime-url "https://bedrock-agentcore.eu-west-2.amazonaws.com/runtimes/..." \
+  --yes
+```
+
+Interactive wizard: enable agent mode, then choose **Bedrock AgentCore** when prompted for the orchestration backend.
+
+CDK writes `ENABLE_AGENTCORE_RUNTIME=True` and the URL into `config/cdk_config.env`; `agent.env` gets `AGENT_ORCHESTRATOR` and `AGENTCORE_RUNTIME_URL` for the Pi Express container.
+
+### Typical deployment order
+
+1. Deploy **doc_redaction** main app (CDK) and note **ExpressServiceEndpoint** (main Express HTTPS URL).  
+2. Deploy **AgentCore** runtime (`agentcore deploy`) and note the URL.  
+3. Set `AGENTCORE_RUNTIME_URL` in `agent.env` / installer and deploy or restart the **Pi Express** agent UI service.
+
+You can deploy the Gradio UI first with `AGENT_ORCHESTRATOR=pi` or `langgraph` and switch to `agentcore` once the runtime URL is known.
+
+## Authentication
+
+| Caller | Typical auth |
+|--------|----------------|
+| **Same AWS account / SDK** | IAM — `bedrock-agentcore:InvokeAgentRuntime` on the runtime ARN |
+| **Gradio container → HTTPS runtime** | Often **OAuth / bearer token** — set `AGENTCORE_API_KEY` for the Gradio → AgentCore HTTP client |
+
+If invoke returns **401** or **403**, check AgentCore inbound auth configuration and that the Gradio task role or bearer token is allowed. See [Authenticate and authorize with Inbound Auth and Outbound Auth](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/) in the AgentCore docs.
+
+## Alternatives (no AgentCore URL)
+
+| `AGENT_ORCHESTRATOR` | When to use |
+|----------------------|-------------|
+| `pi` (default) | Current Pi agent; HF Space; bash + skills |
+| `langgraph` | LangGraph inside the agent container (Docker / ECS); no managed runtime |
+| `agentcore` | Managed AgentCore Runtime on AWS |
+
+## Troubleshooting
+
+| Issue | What to check |
+|-------|----------------|
+| Runtime init timeout / `RuntimeClientError: initialization time exceeded` | Container failed to import `main.py` within 30s. Check CloudWatch `/aws/bedrock-agentcore/runtimes/RedactionAgent_RedactionAgent-ye5Jfw7gKj/` **runtime-logs**. Common cause: packaged bootstrap calling Pi-only modules (`pi_workspace_skills`). Re-run `package_runtime.py` and redeploy. |
+| 403 on `/invocations` | Runtime uses **AWS IAM**; Gradio must call via SigV4 (`boto3` `invoke_agent_runtime`) or set `AGENTCORE_API_KEY` for CUSTOM_JWT. Ensure `AGENT_AWS_PROFILE` / `~/.aws` in the pi-agent container and `bedrock-agentcore:InvokeAgentRuntime` on the runtime ARN. |
+| Agent cannot reach doc_redaction | `DOC_REDACTION_GRADIO_URL` must be the **main app HTTPS URL** for AgentCore (not Service Connect). Check `CloudFrontDistributionURL` (when CloudFront is enabled) or `AgenticDocRedactionBackendUrl` stack output and activity log `Redaction backend for this turn: …` |
+| CDK deploy fails | `cdk bootstrap`; `agentcore deploy -v` for verbose AgentCore errors |
+| `Failed to parse: \`-\`` during **Synthesize CloudFormation** | Windows + path with spaces (e.g. `OneDrive - Lambeth Council`). AgentCore CDK runs `uv` with `shell: true` and unquoted paths; the `-` in the folder name is passed to `uv` as a bogus package. See below. |
+| `hardlink` / `os error 396` during synth | Project on OneDrive; set `UV_LINK_MODE=copy` before deploy |
+| Region errors | AgentCore availability per region; align `aws-targets.json` and Bedrock model region |
+
+### Windows: `Failed to parse: \`-\`` (spaces in project path)
+
+**Cause:** `agentcore deploy` packages Python deps with `uv` during CDK synth. On Windows the CDK subprocess uses a shell without quoting, so a path like:
+
+`C:\Users\Sean\OneDrive - Lambeth Council\...\RedactionAgent`
+
+is split at spaces and `uv` receives a lone `-` argument (from `OneDrive - Lambeth`).
+
+**Fix (pick one):**
+
+1. **Deploy from a short path without spaces** (recommended):
+
+   ```powershell
+   # Copy (not junction) to a local non-synced folder if OneDrive hardlinks also fail
+   xcopy /E /I "...\agent-redact\RedactionAgent" C:\dev\RedactionAgent
+   cd C:\dev\RedactionAgent
+   $env:UV_LINK_MODE = "copy"   # needed if cache/staging still hits OneDrive
+   agentcore deploy
+   ```
+
+2. **Junction** (quick test; staging may still land on OneDrive via the target):
+
+   ```powershell
+   mklink /J C:\dev\RedactionAgent "...\agent-redact\RedactionAgent"
+   cd C:\dev\RedactionAgent
+   $env:UV_LINK_MODE = "copy"
+   agentcore deploy
+   ```
+
+Verified: `node dist/bin/cdk.js synth` succeeds from `C:\dev\RedactionAgent\agentcore\cdk` with `UV_LINK_MODE=copy`; it fails with the `-` parse error from the OneDrive path directly.
+
+Post-deploy reminder from [`cdk/post_cdk_build_quickstart.py`](../../cdk/post_cdk_build_quickstart.py): when `ENABLE_AGENTCORE_RUNTIME=True`, run `agentcore deploy` for this entrypoint and ensure `AGENTCORE_RUNTIME_URL` is set before scaling the Pi agent service.
+
+## References
+
+- [Get started with the AgentCore CLI](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-get-started-cli.html)
+- [HTTP protocol contract (`/invocations`)](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-http-protocol-contract.html)
+- [bedrock-agentcore-sdk-python](https://github.com/aws/bedrock-agentcore-sdk-python)
+- Repo: [`agent_runtime.py`](../shared/agent_runtime.py), [`agent-redact/pi/agent/README.md`](../pi/agent/README.md)

agent-redact/agentcore/agentcore_boto.py

@@ -0,0 +1,47 @@
+"""Shared boto3 helpers for Bedrock AgentCore runtime and harness clients."""
+
+from __future__ import annotations
+
+from agent_runtime import AgentRuntimeError
+
+
+def bedrock_agentcore_client(region: str):
+    import boto3
+    from botocore.exceptions import BotoCoreError, ClientError, NoCredentialsError
+    from pi_agent_config import configure_aws_credentials
+
+    configure_aws_credentials()
+    session = boto3.Session(region_name=region)
+    try:
+        session.client("sts").get_caller_identity()
+    except (ClientError, BotoCoreError, NoCredentialsError) as exc:
+        raise AgentRuntimeError(
+            "AWS credentials are required to invoke AgentCore. "
+            "Set AWS_PROFILE / AGENT_AWS_PROFILE, mount ~/.aws into the pi-agent container, "
+            "or paste session keys under **Agent backend** → **Apply backend**. "
+            "For HTTP runtime auth with CUSTOM_JWT, set AGENTCORE_API_KEY instead."
+        ) from exc
+    return session.client("bedrock-agentcore", region_name=region)
+
+
+def region_from_agentcore_arn(arn: str, *, resource_label: str) -> str:
+    """Return AWS region from a bedrock-agentcore ARN."""
+    normalized = (arn or "").strip()
+    if not normalized.startswith("arn:"):
+        raise AgentRuntimeError(
+            f"Expected an AgentCore {resource_label} ARN, got: {arn!r}"
+        )
+    parts = normalized.split(":")
+    if len(parts) < 6 or parts[2] != "bedrock-agentcore":
+        raise AgentRuntimeError(f"Invalid AgentCore {resource_label} ARN: {arn!r}")
+    region = parts[3].strip()
+    if not region:
+        raise AgentRuntimeError(
+            f"Could not parse region from {resource_label} ARN: {arn!r}"
+        )
+    if f":{resource_label}/" not in normalized:
+        raise AgentRuntimeError(
+            f"ARN must be a {resource_label} resource (arn:...:bedrock-agentcore:...:{resource_label}/...), "
+            f"got: {arn!r}"
+        )
+    return region

agent-redact/agentcore/agentcore_harness_runtime.py

@@ -0,0 +1,270 @@
+"""Bedrock AgentCore Harness client :class:`AgentRuntime` for the Gradio UI."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from collections.abc import Iterator
+from typing import Any
+
+from agent_runtime import AgentRuntime, AgentRuntimeError, AgentStreamEvent
+from agentcore_boto import bedrock_agentcore_client, region_from_agentcore_arn
+
+
+def agentcore_harness_arn() -> str:
+    raw = (os.environ.get("AGENTCORE_HARNESS_ARN") or "").strip()
+    if not raw:
+        raise AgentRuntimeError(
+            "AGENTCORE_HARNESS_ARN is not set. Create a Harness in the AgentCore console "
+            "or use AGENT_ORCHESTRATOR=pi|langgraph|agentcore for other backends."
+        )
+    return raw
+
+
+def agentcore_harness_endpoint() -> str:
+    return (
+        os.environ.get("AGENTCORE_HARNESS_ENDPOINT") or "DEFAULT"
+    ).strip() or "DEFAULT"
+
+
+def harness_runtime_session_id(session_hash: str | None) -> str:
+    """Stable AgentCore Harness session id (must be at least 33 characters)."""
+    base = (session_hash or "default").strip() or "default"
+    digest = hashlib.sha256(base.encode("utf-8")).hexdigest()
+    return f"harness-{digest}"
+
+
+def parse_agentcore_harness_arn(arn: str) -> tuple[str, str]:
+    """Return ``(region, harness_arn)``."""
+    normalized = (arn or "").strip()
+    return region_from_agentcore_arn(normalized, resource_label="harness"), normalized
+
+
+def map_harness_stream_event(event: dict[str, Any]) -> Iterator[AgentStreamEvent]:
+    """Map one ``InvokeHarness`` stream event to normalized Gradio events."""
+    if "runtimeClientError" in event:
+        err = event.get("runtimeClientError") or {}
+        message = str(err.get("message") or err.get("errorMessage") or "Harness error")
+        yield AgentStreamEvent(kind="error", text=message, is_error=True)
+        return
+
+    if "contentBlockDelta" in event:
+        delta = (event.get("contentBlockDelta") or {}).get("delta") or {}
+        text = delta.get("text")
+        if text:
+            yield AgentStreamEvent(kind="text_delta", text=str(text))
+        reasoning = (delta.get("reasoningContent") or {}).get("text")
+        if reasoning:
+            yield AgentStreamEvent(kind="thinking_delta", text=str(reasoning))
+        tool_input = delta.get("toolUse") or {}
+        if isinstance(tool_input, dict) and tool_input.get("input"):
+            yield AgentStreamEvent(
+                kind="status",
+                text=f"Tool input: {json.dumps(tool_input.get('input'), default=str)[:500]}",
+            )
+        return
+
+    if "contentBlockStart" in event:
+        start = (event.get("contentBlockStart") or {}).get("start") or {}
+        tool_use = start.get("toolUse") or {}
+        if isinstance(tool_use, dict) and tool_use.get("name"):
+            name = str(tool_use.get("name") or "tool")
+            tool_id = str(tool_use.get("toolUseId") or "")
+            args = (
+                tool_use.get("input") if isinstance(tool_use.get("input"), dict) else {}
+            )
+            yield AgentStreamEvent(
+                kind="tool_start",
+                tool_name=name,
+                tool_call_id=tool_id or None,
+                tool_args=args,
+                text=name,
+            )
+        return
+
+    if "contentBlockStop" in event:
+        stop = event.get("contentBlockStop") or {}
+        tool_result = stop.get("toolResult") or {}
+        if isinstance(tool_result, dict) and tool_result:
+            name = str(tool_result.get("toolName") or tool_result.get("name") or "tool")
+            content = (
+                tool_result.get("content") or tool_result.get("output") or tool_result
+            )
+            output = (
+                content
+                if isinstance(content, str)
+                else json.dumps(content, default=str)
+            )
+            yield AgentStreamEvent(
+                kind="tool_end",
+                tool_name=name,
+                tool_output=output,
+                is_error="error" in output.lower(),
+            )
+        return
+
+    if "messageStop" in event:
+        stop = event.get("messageStop") or {}
+        reason = str(stop.get("stopReason") or "")
+        if reason:
+            yield AgentStreamEvent(kind="status", text=f"Harness stopped: {reason}")
+        return
+
+    if "messageStart" in event:
+        yield AgentStreamEvent(kind="status", text="Harness message started…")
+        return
+
+
+class AgentCoreHarnessRuntime(AgentRuntime):
+    """Proxy that streams events from a remote Bedrock AgentCore Harness."""
+
+    def __init__(self, *, session_hash: str | None = None) -> None:
+        self._session_hash = session_hash
+        self._running = False
+        self._prompt_stream_depth = 0
+        self._abort_requested = False
+        self._pending_ui_history: list[dict[str, Any]] = []
+        self._pending_prompt_prefix = ""
+
+    @property
+    def orchestrator(self) -> str:
+        return "agentcore-harness"
+
+    @property
+    def running(self) -> bool:
+        return self._running
+
+    @property
+    def prompt_stream_active(self) -> bool:
+        return self._prompt_stream_depth > 0
+
+    @property
+    def abort_requested(self) -> bool:
+        return self._abort_requested
+
+    def start(self) -> None:
+        agentcore_harness_arn()
+        self._running = True
+
+    def close(self) -> None:
+        self._running = False
+
+    def abort(self) -> None:
+        self._abort_requested = True
+        try:
+            region, _arn = parse_agentcore_harness_arn(agentcore_harness_arn())
+            client = bedrock_agentcore_client(region)
+            stop = getattr(client, "stop_runtime_session", None)
+            if callable(stop):
+                stop(runtimeSessionId=harness_runtime_session_id(self._session_hash))
+        except Exception:
+            pass
+
+    def new_session(self) -> None:
+        self._abort_requested = False
+
+    def set_model(self, provider: str, model_id: str) -> dict[str, Any]:
+        os.environ["AGENT_DEFAULT_PROVIDER"] = provider
+        os.environ["AGENT_DEFAULT_MODEL"] = model_id
+        return {"provider": provider, "model": model_id}
+
+    def get_state(self) -> dict[str, Any]:
+        return {
+            "isStreaming": self.prompt_stream_active,
+            "provider": "agentcore-harness",
+            "model": {
+                "provider": "agentcore-harness",
+                "id": agentcore_harness_arn(),
+                "endpoint": agentcore_harness_endpoint(),
+            },
+        }
+
+    def stage_ui_chat_notice(self, label: str, message: str) -> None:
+        text = message.strip()
+        if not text:
+            return
+        self._pending_ui_history.append(
+            {"role": "user", "content": f"_**{label}:**_ {text}"}
+        )
+        self._pending_ui_history.append({"role": "assistant", "content": ""})
+
+    def drain_pending_ui_history(self) -> list[dict[str, Any]]:
+        pending = self._pending_ui_history[:]
+        self._pending_ui_history.clear()
+        return pending
+
+    def stage_prompt_prefix(self, prefix: str) -> None:
+        text = (prefix or "").strip()
+        if text:
+            self._pending_prompt_prefix = f"{text.rstrip()}\n\n"
+
+    def prompt_events(self, message: str) -> Iterator[AgentStreamEvent]:
+        self._prompt_stream_depth += 1
+        self._abort_requested = False
+        try:
+            if not self._running:
+                self.start()
+            harness_arn = agentcore_harness_arn()
+            region, _parsed = parse_agentcore_harness_arn(harness_arn)
+            client = bedrock_agentcore_client(region)
+            invoke = getattr(client, "invoke_harness", None)
+            if not callable(invoke):
+                raise AgentRuntimeError(
+                    "Your boto3 bedrock-agentcore client does not support invoke_harness. "
+                    "Upgrade boto3/botocore in the pi-agent environment."
+                )
+
+            prompt = f"{self._pending_prompt_prefix}{message}"
+            self._pending_prompt_prefix = ""
+            session_id = harness_runtime_session_id(self._session_hash)
+            request: dict[str, Any] = {
+                "harnessArn": harness_arn,
+                "runtimeSessionId": session_id,
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [{"text": prompt}],
+                    }
+                ],
+            }
+            endpoint = agentcore_harness_endpoint()
+            if endpoint and endpoint.upper() != "DEFAULT":
+                request["endpointName"] = endpoint
+
+            yield AgentStreamEvent(kind="status", text="AgentCore Harness started…")
+            try:
+                response = invoke(**request)
+            except Exception as exc:
+                from botocore.exceptions import ClientError
+
+                if isinstance(exc, ClientError):
+                    code = exc.response.get("Error", {}).get("Code", "")
+                    msg = exc.response.get("Error", {}).get("Message", str(exc))
+                    hint = (
+                        " Ensure your IAM identity has bedrock-agentcore:InvokeHarness on "
+                        f"{harness_arn}."
+                    )
+                    raise AgentRuntimeError(f"{code}: {msg}.{hint}") from exc
+                raise AgentRuntimeError(str(exc)) from exc
+
+            stream = response.get("stream") or []
+            assistant_text: list[str] = []
+            for event in stream:
+                if self._abort_requested:
+                    yield AgentStreamEvent(kind="done", text="Agent aborted.")
+                    return
+                if not isinstance(event, dict):
+                    continue
+                for mapped in map_harness_stream_event(event):
+                    if mapped.kind == "text_delta" and mapped.text:
+                        assistant_text.append(mapped.text)
+                    yield mapped
+
+            if assistant_text:
+                yield AgentStreamEvent(
+                    kind="text_snapshot", text="".join(assistant_text)
+                )
+            yield AgentStreamEvent(kind="done", text="Agent finished.")
+        finally:
+            self._prompt_stream_depth = max(0, self._prompt_stream_depth - 1)

agent-redact/agentcore/agentcore_runtime.py

@@ -0,0 +1,406 @@
+"""AgentCore HTTP/SSE client :class:`AgentRuntime` for the Gradio UI."""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+from collections.abc import Iterator
+from typing import Any
+from urllib.parse import unquote, urlparse
+
+import httpx
+from agent_runtime import AgentRuntime, AgentRuntimeError, AgentStreamEvent
+from agentcore_boto import bedrock_agentcore_client, region_from_agentcore_arn
+
+
+def agentcore_runtime_url() -> str:
+    raw = (os.environ.get("AGENTCORE_RUNTIME_URL") or "").strip().rstrip("/")
+    if raw.endswith("/invocations"):
+        raw = raw[: -len("/invocations")].rstrip("/")
+    if not raw:
+        raise AgentRuntimeError(
+            "AGENTCORE_RUNTIME_URL is not set. Deploy AgentCore runtime or use "
+            "AGENT_ORCHESTRATOR=pi|langgraph for local orchestration."
+        )
+    return raw
+
+
+def parse_agentcore_runtime_url(url: str) -> tuple[str, str]:
+    """Return ``(region, agent_runtime_arn)`` from an AgentCore runtime base URL."""
+    normalized = (url or "").strip().rstrip("/")
+    if normalized.endswith("/invocations"):
+        normalized = normalized[: -len("/invocations")].rstrip("/")
+    parsed = urlparse(normalized)
+    host = (parsed.hostname or "").strip()
+    if not host.startswith("bedrock-agentcore."):
+        raise AgentRuntimeError(
+            f"AGENTCORE_RUNTIME_URL must be a bedrock-agentcore HTTPS URL, got: {url!r}"
+        )
+    region = host.removeprefix("bedrock-agentcore.").split(".", 1)[0].strip()
+    if not region:
+        raise AgentRuntimeError(
+            f"Could not parse AWS region from AgentCore URL: {url!r}"
+        )
+
+    prefix = "/runtimes/"
+    path = parsed.path or ""
+    if not path.startswith(prefix):
+        raise AgentRuntimeError(
+            f"AGENTCORE_RUNTIME_URL must include /runtimes/<arn>, got path: {path!r}"
+        )
+    arn = unquote(path[len(prefix) :].strip("/"))
+    if not arn.startswith("arn:"):
+        raise AgentRuntimeError(
+            f"Could not parse runtime ARN from AgentCore URL: {url!r}"
+        )
+    return region_from_agentcore_arn(arn, resource_label="runtime"), arn
+
+
+def _agentcore_api_key() -> str:
+    return (os.environ.get("AGENTCORE_API_KEY") or "").strip()
+
+
+def _bedrock_agentcore_client(region: str):
+    return bedrock_agentcore_client(region)
+
+
+class AgentCoreAgentRuntime(AgentRuntime):
+    """Proxy that streams events from a remote Bedrock AgentCore runtime."""
+
+    def __init__(self, *, session_hash: str | None = None) -> None:
+        self._session_hash = session_hash
+        self._running = False
+        self._prompt_stream_depth = 0
+        self._abort_requested = False
+        self._pending_ui_notices: list[dict[str, Any]] = []
+        self._pending_ui_history: list[dict[str, Any]] = []
+        self._pending_workspace_files: list[dict[str, str]] = []
+        self._sync_workspace_files = True
+
+    @property
+    def orchestrator(self) -> str:
+        return "agentcore"
+
+    @property
+    def running(self) -> bool:
+        return self._running
+
+    @property
+    def prompt_stream_active(self) -> bool:
+        return self._prompt_stream_depth > 0
+
+    @property
+    def abort_requested(self) -> bool:
+        return self._abort_requested
+
+    def start(self) -> None:
+        agentcore_runtime_url()
+        self._running = True
+
+    def close(self) -> None:
+        self._running = False
+
+    def abort(self) -> None:
+        self._abort_requested = True
+
+    def new_session(self) -> None:
+        self._abort_requested = False
+
+    def set_model(self, provider: str, model_id: str) -> dict[str, Any]:
+        os.environ["AGENT_DEFAULT_PROVIDER"] = provider
+        os.environ["AGENT_DEFAULT_MODEL"] = model_id
+        return {"provider": provider, "model": model_id}
+
+    def get_state(self) -> dict[str, Any]:
+        return {
+            "isStreaming": self.prompt_stream_active,
+            "provider": "agentcore",
+            "model": {"provider": "agentcore", "id": agentcore_runtime_url()},
+        }
+
+    def stage_ui_chat_notice(self, label: str, message: str) -> None:
+        text = message.strip()
+        if not text:
+            return
+        self._pending_ui_history.append(
+            {"role": "user", "content": f"_**{label}:**_ {text}"}
+        )
+        self._pending_ui_history.append({"role": "assistant", "content": ""})
+
+    def drain_pending_ui_history(self) -> list[dict[str, Any]]:
+        pending = self._pending_ui_history[:]
+        self._pending_ui_history.clear()
+        return pending
+
+    def stage_workspace_files(self, files: list[dict[str, str]]) -> None:
+        """Queue files to upload into the remote AgentCore session workspace on next invoke."""
+        for item in files:
+            if not isinstance(item, dict):
+                continue
+            relative = str(item.get("relative_path") or item.get("name") or "").strip()
+            encoded = str(item.get("content_base64") or "").strip()
+            if relative and encoded:
+                self._pending_workspace_files.append(
+                    {"relative_path": relative, "content_base64": encoded}
+                )
+
+    def set_sync_workspace_files(self, enabled: bool) -> None:
+        self._sync_workspace_files = bool(enabled)
+
+    def _write_local_workspace_file(
+        self, relative_path: str, content_base64: str
+    ) -> None:
+        if not self._session_hash:
+            return
+        from session_workspace import session_workspace_dir
+
+        root = session_workspace_dir(self._session_hash).resolve()
+        dest = (root / relative_path).resolve()
+        try:
+            dest.relative_to(root)
+        except ValueError:
+            return
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_bytes(base64.b64decode(content_base64, validate=True))
+
+    def prompt_events(self, message: str) -> Iterator[AgentStreamEvent]:
+        self._prompt_stream_depth += 1
+        self._abort_requested = False
+        try:
+            if not self._running:
+                self.start()
+            payload: dict[str, Any] = {
+                "prompt": message,
+                "session_hash": self._session_hash or "",
+            }
+            if self._pending_workspace_files:
+                payload["workspace_files"] = self._pending_workspace_files[:]
+                self._pending_workspace_files.clear()
+            if self._sync_workspace_files:
+                payload["sync_workspace_files"] = True
+            from agentcore_workspace_bridge import build_agentcore_invoke_runtime_config
+
+            runtime_config = build_agentcore_invoke_runtime_config()
+            if runtime_config:
+                payload["runtime_config"] = runtime_config
+            yield AgentStreamEvent(kind="status", text="AgentCore runtime started…")
+            if _agentcore_api_key():
+                yield from self._prompt_events_httpx(payload)
+            else:
+                yield from self._prompt_events_boto3(payload)
+            yield AgentStreamEvent(kind="done", text="Agent finished.")
+        except httpx.HTTPError as exc:
+            raise AgentRuntimeError(str(exc)) from exc
+        finally:
+            self._prompt_stream_depth = max(0, self._prompt_stream_depth - 1)
+
+    def _prompt_events_httpx(
+        self, payload: dict[str, Any]
+    ) -> Iterator[AgentStreamEvent]:
+        url = f"{agentcore_runtime_url()}/invocations"
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "text/event-stream",
+            "Authorization": f"Bearer {_agentcore_api_key()}",
+        }
+        timeout = httpx.Timeout(connect=30.0, read=1800.0, write=30.0, pool=30.0)
+        with httpx.Client(timeout=timeout) as client:
+            with client.stream("POST", url, json=payload, headers=headers) as response:
+                response.raise_for_status()
+                yield from self._iter_sse_response(response.iter_lines())
+
+    def _prompt_events_boto3(
+        self, payload: dict[str, Any]
+    ) -> Iterator[AgentStreamEvent]:
+        from botocore.exceptions import ClientError
+
+        region, runtime_arn = parse_agentcore_runtime_url(agentcore_runtime_url())
+        client = _bedrock_agentcore_client(region)
+        body = json.dumps(payload).encode("utf-8")
+        try:
+            response = client.invoke_agent_runtime(
+                agentRuntimeArn=runtime_arn,
+                payload=body,
+                contentType="application/json",
+                accept="text/event-stream",
+            )
+        except ClientError as exc:
+            code = exc.response.get("Error", {}).get("Code", "")
+            message = exc.response.get("Error", {}).get("Message", str(exc))
+            hint = (
+                " Ensure your IAM identity has bedrock-agentcore:InvokeAgentRuntime on "
+                f"{runtime_arn}."
+            )
+            if "initialization" in message.lower() or code == "RuntimeClientError":
+                hint += (
+                    " The runtime container failed to start (import error or slow init). "
+                    "Check CloudWatch log group "
+                    f"/aws/bedrock-agentcore/runtimes/{runtime_arn.rsplit('/', 1)[-1]}/"
+                    " then re-run package_runtime.py and agentcore deploy."
+                )
+            raise AgentRuntimeError(f"{code}: {message}.{hint}") from exc
+
+        status_code = int(response.get("statusCode") or 200)
+        if status_code >= 400:
+            raise AgentRuntimeError(
+                f"AgentCore invoke failed with HTTP {status_code} for runtime {runtime_arn}."
+            )
+
+        stream = response.get("response")
+        if stream is None:
+            return
+        if hasattr(stream, "iter_lines"):
+            yield from self._iter_sse_response(
+                (
+                    line.decode("utf-8", errors="replace")
+                    if isinstance(line, (bytes, bytearray))
+                    else str(line)
+                )
+                for line in stream.iter_lines()
+            )
+            return
+        raw = stream.read() if hasattr(stream, "read") else stream
+        if isinstance(raw, str):
+            raw_bytes = raw.encode("utf-8")
+        else:
+            raw_bytes = bytes(raw or b"")
+        content_type = str(response.get("contentType") or "").lower()
+        if "event-stream" in content_type or raw_bytes.strip().startswith(b"data:"):
+            yield from self._iter_sse_lines(
+                line.decode("utf-8", errors="replace")
+                for line in raw_bytes.splitlines()
+            )
+        else:
+            yield from self._iter_json_response(raw_bytes)
+
+    def _iter_sse_response(self, lines: Iterator[str]) -> Iterator[AgentStreamEvent]:
+        for line in lines:
+            if self._abort_requested:
+                yield AgentStreamEvent(kind="done", text="Agent aborted.")
+                return
+            if not line or not line.startswith("data:"):
+                continue
+            data = line[5:].strip()
+            if not data or data == "[DONE]":
+                continue
+            try:
+                event = json.loads(data)
+            except json.JSONDecodeError:
+                yield AgentStreamEvent(kind="text_delta", text=data)
+                continue
+            yield from self._map_agentcore_event(event)
+
+    def _iter_sse_lines(self, lines: Iterator[str]) -> Iterator[AgentStreamEvent]:
+        yield from self._iter_sse_response(lines)
+
+    def _iter_json_response(self, raw: bytes) -> Iterator[AgentStreamEvent]:
+        if not raw.strip():
+            return
+        try:
+            payload = json.loads(raw.decode("utf-8"))
+        except json.JSONDecodeError:
+            yield AgentStreamEvent(
+                kind="text_snapshot", text=raw.decode("utf-8", errors="replace")
+            )
+            return
+        if isinstance(payload, dict):
+            if payload.get("type") == "error":
+                yield AgentStreamEvent(
+                    kind="error",
+                    text=str(payload.get("message") or "AgentCore error"),
+                    is_error=True,
+                )
+                return
+            if "result" in payload:
+                yield AgentStreamEvent(
+                    kind="text_snapshot", text=str(payload["result"])
+                )
+                return
+            yield AgentStreamEvent(
+                kind="text_snapshot",
+                text=json.dumps(payload, default=str),
+            )
+        else:
+            yield AgentStreamEvent(kind="text_snapshot", text=str(payload))
+
+    def _map_agentcore_event(self, event: dict[str, Any]) -> Iterator[AgentStreamEvent]:
+        event_type = str(event.get("type") or "")
+        if event_type == "agent_start":
+            yield AgentStreamEvent(kind="status", text="Agent started…")
+        elif event_type == "agent_end":
+            yield AgentStreamEvent(
+                kind="status", text=str(event.get("message") or "Agent finished.")
+            )
+        elif event_type == "status":
+            yield AgentStreamEvent(kind="status", text=str(event.get("message") or ""))
+        elif event_type == "error":
+            yield AgentStreamEvent(
+                kind="error",
+                text=str(event.get("message") or "AgentCore error"),
+                is_error=True,
+            )
+        elif event_type == "workspace_file":
+            relative = str(event.get("relative_path") or "").strip()
+            encoded = str(event.get("content_base64") or "").strip()
+            if relative and encoded:
+                try:
+                    self._write_local_workspace_file(relative, encoded)
+                    yield AgentStreamEvent(
+                        kind="status",
+                        text=f"Downloaded `{relative}` from AgentCore workspace.",
+                    )
+                    if relative.lower().endswith("_redacted.pdf"):
+                        yield AgentStreamEvent(kind="workspace_sync")
+                except (OSError, ValueError) as exc:
+                    yield AgentStreamEvent(
+                        kind="status",
+                        text=f"Could not save `{relative}` locally: {exc}",
+                    )
+        elif event_type == "message_update":
+            role = str(event.get("role") or "")
+            if role == "tool":
+                tool_name = str(event.get("tool_name") or "tool")
+                content = event.get("content")
+                output = (
+                    content
+                    if isinstance(content, str)
+                    else json.dumps(content, default=str)
+                )
+                is_error = "error" in output.lower() or "not found" in output.lower()
+                yield AgentStreamEvent(
+                    kind="tool_end",
+                    tool_name=tool_name,
+                    tool_output=output,
+                    is_error=is_error,
+                )
+            else:
+                content = event.get("content")
+                text = (
+                    content
+                    if isinstance(content, str)
+                    else json.dumps(content, default=str)
+                )
+                tool_calls = event.get("tool_calls") or []
+                if isinstance(tool_calls, list):
+                    for call in tool_calls:
+                        if not isinstance(call, dict):
+                            continue
+                        name = str(call.get("name") or "tool")
+                        args = (
+                            call.get("args")
+                            if isinstance(call.get("args"), dict)
+                            else {}
+                        )
+                        yield AgentStreamEvent(
+                            kind="tool_start",
+                            tool_name=name,
+                            tool_args=args,
+                            text=name,
+                        )
+                if text.strip():
+                    yield AgentStreamEvent(kind="text_snapshot", text=text)
+        else:
+            yield AgentStreamEvent(
+                kind="status", text=json.dumps(event, default=str)[:500]
+            )

agent-redact/agentcore/agentcore_workspace_bridge.py

@@ -0,0 +1,359 @@
+"""Bridge local Gradio session workspaces to AgentCore runtime invokes."""
+
+from __future__ import annotations
+
+import base64
+import os
+from pathlib import Path
+from typing import Any
+
+from session_workspace import session_workspace_dir
+
+_SKIP_UPLOAD_PREFIXES = (
+    "preview/",
+    ".pi/preview/",
+    "output_final_download/",
+)
+_DEFAULT_MAX_BYTES = 8 * 1024 * 1024
+_DEFAULT_MAX_FILES = 80
+
+
+def max_upload_bytes() -> int:
+    raw = (os.environ.get("AGENTCORE_MAX_UPLOAD_BYTES") or "").strip()
+    if raw.isdigit():
+        return int(raw)
+    return _DEFAULT_MAX_BYTES
+
+
+def max_upload_files() -> int:
+    raw = (os.environ.get("AGENTCORE_MAX_UPLOAD_FILES") or "").strip()
+    if raw.isdigit():
+        return max(1, int(raw))
+    return _DEFAULT_MAX_FILES
+
+
+def discover_session_document_name(session_hash: str) -> str | None:
+    """Return the newest PDF basename at the session workspace root."""
+    root = session_workspace_dir(session_hash)
+    if not root.is_dir():
+        return None
+    candidates: list[tuple[float, str]] = []
+    for path in root.glob("*.pdf"):
+        if not path.is_file():
+            continue
+        try:
+            candidates.append((path.stat().st_mtime, path.name))
+        except OSError:
+            continue
+    if not candidates:
+        return None
+    return max(candidates, key=lambda item: item[0])[1]
+
+
+def _should_upload_relative_path(relative: str) -> bool:
+    rel = relative.replace("\\", "/").lstrip("/")
+    if any(rel.startswith(prefix) for prefix in _SKIP_UPLOAD_PREFIXES):
+        return False
+    if rel.lower().endswith(".pdf") and "/" not in rel:
+        return True
+    return rel.startswith("redact/")
+
+
+def _upload_priority(relative: str) -> tuple[int, str]:
+    rel = relative.replace("\\", "/")
+    if rel.lower().endswith("_review_file.csv"):
+        return (0, rel)
+    if rel.lower().endswith(".pdf") and "/" not in rel:
+        return (1, rel)
+    if rel.startswith("redact/") and rel.lower().endswith("_redacted.pdf"):
+        return (2, rel)
+    if rel.startswith("redact/"):
+        return (3, rel)
+    return (9, rel)
+
+
+def collect_session_files_for_agentcore_upload(
+    session_hash: str,
+    *,
+    document_name: str | None = None,
+) -> list[dict[str, str]]:
+    """
+    Collect local session files to seed the remote AgentCore workspace.
+
+    Includes the source PDF and everything under ``redact/`` (review CSVs, OCR
+    exports, etc.) so follow-up turns work after a runtime cold start.
+    """
+    root = session_workspace_dir(session_hash).resolve()
+    if not root.is_dir():
+        return []
+
+    limit_bytes = max_upload_bytes()
+    limit_files = max_upload_files()
+    doc_name = (
+        document_name or discover_session_document_name(session_hash) or ""
+    ).strip()
+
+    candidates: list[tuple[tuple[int, str], Path]] = []
+    for path in sorted(root.rglob("*")):
+        if not path.is_file():
+            continue
+        try:
+            relative = path.relative_to(root).as_posix()
+        except ValueError:
+            continue
+        if doc_name and relative == doc_name:
+            candidates.append(((1, relative), path))
+            continue
+        if not _should_upload_relative_path(relative):
+            continue
+        candidates.append((_upload_priority(relative), path))
+
+    if doc_name:
+        doc_path = (root / doc_name).resolve()
+        if doc_path.is_file() and all(path != doc_path for _, path in candidates):
+            candidates.append(((1, doc_name), doc_path))
+
+    candidates.sort(key=lambda item: item[0])
+    staged: list[dict[str, str]] = []
+    skipped_large: list[str] = []
+
+    for _, path in candidates:
+        if len(staged) >= limit_files:
+            break
+        try:
+            size = path.stat().st_size
+        except OSError:
+            continue
+        if size > limit_bytes:
+            skipped_large.append(path.relative_to(root).as_posix())
+            continue
+        try:
+            payload = path.read_bytes()
+        except OSError:
+            continue
+        staged.append(
+            {
+                "relative_path": path.relative_to(root).as_posix(),
+                "content_base64": base64.b64encode(payload).decode("ascii"),
+            }
+        )
+
+    if skipped_large and staged:
+        staged.append(
+            {
+                "relative_path": ".agentcore_upload_skipped.txt",
+                "content_base64": base64.b64encode(
+                    (
+                        "Some local files were not uploaded (over "
+                        f"{limit_bytes:,} bytes):\n" + "\n".join(skipped_large[:20])
+                    ).encode("utf-8")
+                ).decode("ascii"),
+            }
+        )
+    return staged
+
+
+def _find_review_csv_paths(session_hash: str) -> list[str]:
+    root = session_workspace_dir(session_hash)
+    if not root.is_dir():
+        return []
+    found: list[str] = []
+    for path in sorted(root.rglob("*_review_file.csv")):
+        if not path.is_file():
+            continue
+        try:
+            found.append(path.relative_to(root).as_posix())
+        except ValueError:
+            continue
+    return found[:5]
+
+
+def build_agentcore_followup_context(
+    session_hash: str,
+    history: list[dict[str, Any]] | None = None,
+) -> str:
+    """Prompt prefix so follow-ups continue Pass 1 instead of restarting upload flow."""
+    doc = discover_session_document_name(session_hash)
+    review_csvs = _find_review_csv_paths(session_hash)
+    lines = [
+        "**AgentCore follow-up (mandatory):** Pass 1 redaction already ran in this "
+        "session. The local UI synced workspace artifacts into your session folder "
+        "before this message — call `list_workspace_files` first.",
+        "Do **not** ask the user to re-upload the PDF unless `list_workspace_files` "
+        "is empty after sync.",
+        "Prefer editing the existing `*_review_file.csv` and running `review_apply` "
+        "again (or `verify_coverage` first) rather than a full new `doc_redact`.",
+    ]
+    if doc:
+        lines.append(f"**Source document:** `{doc}`")
+        lines.append(f"**Redaction tree:** `redact/{doc}/`")
+    if review_csvs:
+        lines.append(
+            "**Review CSV(s):** " + ", ".join(f"`{path}`" for path in review_csvs)
+        )
+    if history:
+        excerpt = _format_history_excerpt(history)
+        if excerpt:
+            lines.append("**Prior chat (UI, for context):**\n" + excerpt)
+    return "\n".join(lines) + "\n\n"
+
+
+def _format_history_excerpt(
+    history: list[dict[str, Any]],
+    *,
+    max_messages: int = 10,
+    max_chars: int = 6000,
+) -> str:
+    chunks: list[str] = []
+    total = 0
+    for message in history[-max_messages:]:
+        role = str(message.get("role") or "user").strip()
+        content = str(message.get("content") or "").strip()
+        if not content:
+            continue
+        line = f"- **{role}:** {content[:1500]}"
+        if total + len(line) > max_chars:
+            break
+        chunks.append(line)
+        total += len(line)
+    return "\n".join(chunks)
+
+
+_CLOUDFRONT_OVERLAY_KEYS = (
+    "DOC_REDACTION_GRADIO_URL",
+    "DOC_REDACTION_AUTH_TOKEN",
+    "DOC_REDACTION_AUTH_COOKIE_NAME",
+    # AgentCore runtime URL + model are created/finalised in deploy phase 2 and
+    # re-uploaded to agent.env in S3; the container's task env is fixed at synth,
+    # so pick them up here on (re)start rather than requiring a stack update.
+    "AGENTCORE_RUNTIME_URL",
+    "AGENT_DEFAULT_MODEL",
+    "AGENT_DEFAULT_PROVIDER",
+)
+_cloudfront_overlay_applied = False
+
+
+def apply_agentcore_cloudfront_config_overlay() -> dict[str, str]:
+    """Overlay post-deploy AgentCore settings from S3 into ``os.environ`` (AgentCore only).
+
+    The Pi Express task definition is fixed at synth time and cannot carry values
+    that only exist after deploy phase 2 — the (later-created) CloudFront
+    domain/magic-link token and the AgentCore runtime URL/model — and injecting
+    them into the task env would create a dependency cycle. So when running as the
+    AgentCore orchestrator, we fetch the post-deploy ``agent.env`` from S3 and
+    override those keys (see ``_CLOUDFRONT_OVERLAY_KEYS``): the CloudFront backend
+    URL + token cookie for ``build_agentcore_invoke_runtime_config``, plus
+    ``AGENTCORE_RUNTIME_URL`` / ``AGENT_DEFAULT_MODEL`` so the running container
+    reaches the runtime and shows the correct model without a stack update.
+
+    Controlled by ``DOC_REDACTION_CONFIG_S3_BUCKET`` / ``DOC_REDACTION_CONFIG_S3_KEY``
+    (set by CDK on every Express deploy). The overlay runs whenever that S3 config
+    source is present and will **promote** the container to ``AGENT_ORCHESTRATOR=agentcore``
+    if the S3 ``agent.env`` says so — this lets deploy phase 2 (which re-uploads
+    ``agent.env`` and recycles the service) flip an already-running container to
+    AgentCore without a stack update. It never demotes an already-agentcore
+    container and is a no-op when the S3 config does not select AgentCore.
+    Best-effort and idempotent; returns the keys it overrode.
+    """
+    global _cloudfront_overlay_applied
+    applied: dict[str, str] = {}
+    if _cloudfront_overlay_applied:
+        return applied
+    already_agentcore = (
+        os.environ.get("AGENT_ORCHESTRATOR") or ""
+    ).strip().lower() == "agentcore"
+    bucket = (os.environ.get("DOC_REDACTION_CONFIG_S3_BUCKET") or "").strip()
+    if not bucket:
+        # No post-deploy S3 config source; rely solely on the task env.
+        _cloudfront_overlay_applied = True
+        return applied
+    key = (os.environ.get("DOC_REDACTION_CONFIG_S3_KEY") or "agent.env").strip()
+
+    try:
+        import boto3
+
+        region = (
+            os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or None
+        )
+        body = (
+            boto3.client("s3", region_name=region)
+            .get_object(Bucket=bucket, Key=key)["Body"]
+            .read()
+            .decode("utf-8", "replace")
+        )
+    except Exception as exc:  # noqa: BLE001 - best effort, keep the UI booting
+        print(f"AgentCore config overlay skipped ({bucket}/{key}): {exc}")
+        _cloudfront_overlay_applied = True
+        return applied
+
+    parsed: dict[str, str] = {}
+    for raw_line in body.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        name, _, value = line.partition("=")
+        parsed[name.strip()] = value.strip().strip('"').strip("'")
+
+    # Only act when this deployment actually selects AgentCore (task env already
+    # says so, or the S3 config does). This keeps the overlay a no-op for pi /
+    # langgraph deployments that share the same config bucket.
+    s3_agentcore = (
+        parsed.get("AGENT_ORCHESTRATOR") or ""
+    ).strip().lower() == "agentcore"
+    if not (already_agentcore or s3_agentcore):
+        _cloudfront_overlay_applied = True
+        return applied
+
+    if s3_agentcore and not already_agentcore:
+        os.environ["AGENT_ORCHESTRATOR"] = "agentcore"
+        applied["AGENT_ORCHESTRATOR"] = "agentcore"
+
+    for name in _CLOUDFRONT_OVERLAY_KEYS:
+        value = parsed.get(name)
+        if value:
+            os.environ[name] = value
+            applied[name] = value
+
+    _cloudfront_overlay_applied = True
+    if applied:
+        print(
+            "Applied AgentCore config overlay from S3 for: "
+            + ", ".join(sorted(applied))
+        )
+    return applied
+
+
+def build_agentcore_invoke_runtime_config() -> dict[str, str]:
+    """
+    Backend settings from the local Gradio process for each AgentCore invoke.
+
+    Overrides ``agentcore.env`` on the AWS runtime so the remote agent uses the same
+    ``DOC_REDACTION_GRADIO_URL`` shown in the Pi UI (not a baked-in HF Space default).
+    """
+    from redaction_prompt import doc_redaction_gradio_url
+
+    url = doc_redaction_gradio_url().strip().rstrip("/")
+    config: dict[str, str] = {}
+    if url:
+        config["DOC_REDACTION_GRADIO_URL"] = url
+    for key in (
+        "DOC_REDACTION_GRADIO_AUTH_USER",
+        "DOC_REDACTION_GRADIO_AUTH_PASSWORD",
+        # CloudFront magic-link: the AgentCore runtime runs outside the VPC and
+        # reaches doc_redaction through CloudFront, so it must send the token
+        # cookie on every request (Service Connect is not reachable from it).
+        "DOC_REDACTION_AUTH_TOKEN",
+        "DOC_REDACTION_AUTH_COOKIE_NAME",
+        "AGENT_DEFAULT_OCR_METHOD",
+        "AGENT_DEFAULT_PII_METHOD",
+    ):
+        value = (os.environ.get(key) or "").strip()
+        if value:
+            config[key] = value
+    if "hf.space" in url.lower():
+        token = (
+            os.environ.get("HF_TOKEN") or os.environ.get("DOC_REDACTION_HF_TOKEN") or ""
+        ).strip()
+        if token:
+            config["HF_TOKEN"] = token
+    return config

agent-redact/agentcore/bundle_support/session_workspace.py

@@ -0,0 +1,50 @@
+"""Gradio-free session workspace helpers for AgentCore runtime bundles."""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+
+_SESSION_ID_RE = re.compile(r"[^a-zA-Z0-9_@.+-]+")
+
+
+def workspace_base_dir() -> Path:
+    raw = (os.environ.get("AGENT_WORKSPACE_DIR") or "").strip()
+    if raw:
+        path = Path(raw)
+    else:
+        path = Path("/tmp/agentcore-workspace")
+    path.mkdir(parents=True, exist_ok=True)
+    return path.resolve()
+
+
+def session_workspace_enabled() -> bool:
+    raw = os.environ.get("AGENT_SESSION_WORKSPACE", "").strip().lower()
+    if raw in {"0", "false", "no", "off"}:
+        return False
+    return True
+
+
+def sanitize_session_id(raw: str) -> str:
+    cleaned = _SESSION_ID_RE.sub("_", (raw or "").strip())[:128].strip("_")
+    return cleaned or "default"
+
+
+def session_workspace_dir(session_hash: str) -> Path:
+    base = workspace_base_dir().resolve()
+    if not session_workspace_enabled():
+        return base
+    safe_id = sanitize_session_id(session_hash)
+    candidate = (base / safe_id).resolve()
+    try:
+        candidate.relative_to(base)
+    except ValueError:
+        return (base / "default").resolve()
+    return candidate
+
+
+def ensure_session_workspace(session_hash: str) -> Path:
+    workspace = session_workspace_dir(session_hash)
+    workspace.mkdir(parents=True, exist_ok=True)
+    return workspace

agent-redact/agentcore/entrypoint.py

@@ -0,0 +1,39 @@
+"""Bedrock AgentCore runtime entrypoint wrapping the LangGraph redaction agent."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_AGENTCORE_DIR = Path(__file__).resolve().parent
+_AGENT_REDACT = _AGENTCORE_DIR.parent
+_REPO_ROOT = _AGENT_REDACT.parent
+for path in (
+    _REPO_ROOT,
+    _AGENT_REDACT,
+    _AGENT_REDACT / "shared",
+    _AGENT_REDACT / "pi",
+    _AGENTCORE_DIR,
+):
+    text = str(path)
+    if text not in sys.path:
+        sys.path.insert(0, text)
+
+from invoke_agent import bootstrap_runtime_env, invoke_redaction_agent  # noqa: E402
+
+bootstrap_runtime_env(_REPO_ROOT)
+
+from bedrock_agentcore import BedrockAgentCoreApp  # noqa: E402
+
+app = BedrockAgentCoreApp()
+
+
+@app.entrypoint
+async def handler(request: dict):
+    """Stream LangGraph agent events for one user prompt."""
+    async for event in invoke_redaction_agent(request):
+        yield event
+
+
+if __name__ == "__main__":
+    app.run()

agent-redact/agentcore/harness_input_bridge.py

@@ -0,0 +1,111 @@
+"""Upload task inputs for AgentCore Harness (S3 + presigned URL prompt prefix)."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from urllib.parse import urlparse
+
+from session_workspace import session_workspace_dir
+
+
+def _parse_s3_uri(uri: str) -> tuple[str, str]:
+    parsed = urlparse((uri or "").strip())
+    if parsed.scheme != "s3" or not parsed.netloc:
+        raise ValueError(f"Invalid S3 URI: {uri!r}")
+    prefix = (parsed.path or "").lstrip("/")
+    if prefix and not prefix.endswith("/"):
+        prefix += "/"
+    return parsed.netloc, prefix
+
+
+def harness_s3_input_uri(session_hash: str, file_name: str) -> tuple[str, str, str]:
+    """
+    Return ``(bucket, key, s3_uri)`` for a harness input object.
+
+    Uses ``AGENTCORE_HARNESS_S3_INPUT_PREFIX`` (``s3://bucket/prefix/``) when set,
+    otherwise ``s3://{S3_OUTPUTS_BUCKET}/harness-inputs/{session_hash}/``.
+    """
+    explicit = (os.environ.get("AGENTCORE_HARNESS_S3_INPUT_PREFIX") or "").strip()
+    if explicit:
+        bucket, prefix = _parse_s3_uri(explicit)
+    else:
+        bucket = (os.environ.get("S3_OUTPUTS_BUCKET") or "").strip()
+        if not bucket:
+            raise ValueError(
+                "Set AGENTCORE_HARNESS_S3_INPUT_PREFIX or S3_OUTPUTS_BUCKET for harness file upload."
+            )
+        safe_session = (session_hash or "default").strip().replace("/", "_")[:128]
+        prefix = f"harness-inputs/{safe_session}/"
+    key = f"{prefix}{Path(file_name).name}"
+    return bucket, key, f"s3://{bucket}/{key}"
+
+
+def build_harness_document_prompt_prefix(
+    session_hash: str,
+    document_name: str,
+) -> str | None:
+    """
+    Upload the task PDF to S3 and return a prompt prefix for the Harness to fetch it.
+
+    Returns ``None`` when upload is disabled or the file is missing.
+    """
+    if not document_name:
+        return None
+    run_aws = (os.environ.get("RUN_AWS_FUNCTIONS") or "").strip().lower() in {
+        "1",
+        "true",
+        "yes",
+        "on",
+    }
+    if (
+        not run_aws
+        and not (os.environ.get("AGENTCORE_HARNESS_S3_INPUT_PREFIX") or "").strip()
+    ):
+        return (
+            "**Harness file bridge:** RUN_AWS_FUNCTIONS is off and "
+            "AGENTCORE_HARNESS_S3_INPUT_PREFIX is unset — upload the document to the Harness "
+            "workspace manually or enable S3 upload."
+        )
+
+    root = session_workspace_dir(session_hash)
+    src = root / document_name
+    if not src.is_file():
+        return None
+
+    try:
+        import boto3
+        from botocore.exceptions import BotoCoreError, ClientError
+        from pi_agent_config import configure_aws_credentials
+
+        configure_aws_credentials()
+        bucket, key, s3_uri = harness_s3_input_uri(session_hash, document_name)
+        region = (
+            os.environ.get("AWS_REGION")
+            or os.environ.get("AWS_DEFAULT_REGION")
+            or "eu-west-2"
+        )
+        client = boto3.client("s3", region_name=region)
+        client.upload_file(str(src), bucket, key)
+        presigned = client.generate_presigned_url(
+            "get_object",
+            Params={"Bucket": bucket, "Key": key},
+            ExpiresIn=int(os.environ.get("AGENTCORE_HARNESS_PRESIGN_SECONDS", "3600")),
+        )
+    except (BotoCoreError, ClientError, ValueError, OSError) as exc:
+        return (
+            f"**Harness file bridge error:** Could not upload `{document_name}` to S3 ({exc}). "
+            "Place the file on the Harness workspace mount or fix AWS permissions."
+        )
+
+    mount_path = (
+        os.environ.get("AGENTCORE_HARNESS_S3_MOUNT_PATH") or "/tmp/workspace"
+    ).rstrip("/")
+    dest = f"{mount_path}/{Path(document_name).name}"
+    return (
+        f"**Harness input file (download before Pass 1):**\n"
+        f"- S3 object: `{s3_uri}`\n"
+        f"- Presigned URL (expires in 1h): {presigned}\n"
+        f"- Save to Harness workspace as: `{dest}`\n"
+        f"- Example: `curl -fsSL -o {dest!r} '<presigned-url>'` then use `{dest}` as INPUT_PATH.\n"
+    )

agent-redact/agentcore/invoke_agent.py

@@ -0,0 +1,288 @@
+"""Shared AgentCore invoke logic (monorepo entrypoint + packaged runtime)."""
+
+from __future__ import annotations
+
+import os
+import sys
+from collections.abc import AsyncIterator
+from pathlib import Path
+
+from session_store import (
+    append_turn,
+    clear_session,
+    get_messages,
+    stringify_message_content,
+)
+
+
+def configure_import_paths(app_root: Path | None = None) -> tuple[Path, Path]:
+    """
+    Ensure imports resolve in the monorepo or a packaged AgentCore app folder.
+
+    Returns ``(repo_root, agent_redact_root)`` for bootstrap_pi_config.
+    """
+    root = (app_root or Path(__file__).resolve().parent).resolve()
+    agent_redact = root
+    shared_dir = root / "shared"
+    for path in (root, agent_redact, shared_dir):
+        text = str(path)
+        if text not in sys.path:
+            sys.path.insert(0, text)
+
+    repo_root = root
+    if (root / "agent-redact").is_dir():
+        repo_root = root
+        agent_redact = root / "agent-redact"
+    elif (
+        root.name == "RedactionAgent" and (root.parent.parent / "agent-redact").is_dir()
+    ):
+        repo_root = root.parent.parent.parent
+        agent_redact = repo_root / "agent-redact"
+    return repo_root, agent_redact
+
+
+def bootstrap_runtime_env(app_root: Path) -> None:
+    """
+    Lightweight env setup for AgentCore (no Pi skills sync or monorepo ``tools/``).
+
+    Full :func:`bootstrap_pi_config.ensure_pi_config_env` pulls in ``pi_workspace_skills``
+    and repo ``skills/`` — not vendored in the CodeZip bundle and will fail or stall
+    runtime init on AWS.
+    """
+    from dotenv import load_dotenv
+
+    root = app_root.resolve()
+    for env_name in ("agentcore.env", ".env"):
+        env_file = root / env_name
+        if env_file.is_file():
+            load_dotenv(env_file, override=False)
+
+    os.environ.setdefault("AGENT_WORKSPACE_DIR", "/tmp/agentcore-workspace")
+    os.environ.setdefault("AGENT_REDACTION_SPLIT_BACKEND", "true")
+    os.environ.setdefault("AGENT_DEFAULT_PROVIDER", "amazon-bedrock")
+    os.environ.setdefault("AGENT_DEFAULT_MODEL", "anthropic.claude-sonnet-4-6")
+    os.environ.setdefault(
+        "AWS_REGION", os.environ.get("AWS_DEFAULT_REGION", "eu-west-2")
+    )
+    os.environ.setdefault("AWS_DEFAULT_REGION", os.environ["AWS_REGION"])
+    Path(os.environ["AGENT_WORKSPACE_DIR"]).mkdir(parents=True, exist_ok=True)
+
+
+INVOKE_RUNTIME_CONFIG_KEYS = frozenset(
+    {
+        "DOC_REDACTION_GRADIO_URL",
+        "DOC_REDACTION_GRADIO_AUTH_USER",
+        "DOC_REDACTION_GRADIO_AUTH_PASSWORD",
+        # CloudFront magic-link cookie: the AgentCore runtime runs outside the VPC
+        # and reaches doc_redaction through CloudFront, so the token forwarded by
+        # build_agentcore_invoke_runtime_config must be applied here or every
+        # backend request hits the login wall ("credentials were not provided").
+        "DOC_REDACTION_AUTH_TOKEN",
+        "DOC_REDACTION_AUTH_COOKIE_NAME",
+        "AGENT_DEFAULT_OCR_METHOD",
+        "AGENT_DEFAULT_PII_METHOD",
+        "HF_TOKEN",
+        "DOC_REDACTION_HF_TOKEN",
+    }
+)
+
+
+def apply_invoke_runtime_config(request: dict) -> None:
+    """
+      Apply per-invoke backend settings from the Gradio UI (overrides agentcore.env).
+
+      The AgentCore runtime on AWS has its own ``agentcore.env``; without this, a
+    deployed HF Space URL can win over the operator's local ``agent.env``.
+    """
+    raw = request.get("runtime_config") or request.get("runtime_env") or {}
+    if not isinstance(raw, dict):
+        return
+    for key in INVOKE_RUNTIME_CONFIG_KEYS:
+        value = raw.get(key)
+        if value is None:
+            continue
+        text = str(value).strip()
+        if text:
+            os.environ[key] = text
+
+
+async def invoke_redaction_agent(request: dict) -> AsyncIterator[dict]:
+    """Stream LangGraph agent events for one user prompt (multi-turn per session_hash)."""
+    # Must run before LangChain imports (OpenInference patch order).
+    try:
+        from eval.arize_monitoring import setup_arize_ax_tracing
+
+        setup_arize_ax_tracing()
+    except ImportError:
+        pass
+
+    from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
+    from workspace_sync import (
+        apply_workspace_files,
+        collect_workspace_files_for_sync,
+    )
+
+    apply_invoke_runtime_config(request)
+
+    prompt = str(request.get("prompt") or request.get("message") or "").strip()
+    session_hash = str(request.get("session_hash") or "").strip() or None
+    if request.get("new_session"):
+        clear_session(session_hash)
+
+    if not prompt:
+        yield {"type": "error", "message": "prompt is required"}
+        return
+
+    incoming_files = request.get("workspace_files") or []
+    if isinstance(incoming_files, list) and incoming_files:
+        written = apply_workspace_files(session_hash, incoming_files)
+        if written:
+            yield {
+                "type": "status",
+                "message": f"Synced {len(written)} file(s) into AgentCore workspace.",
+            }
+
+    backend_url = (os.environ.get("DOC_REDACTION_GRADIO_URL") or "").strip().rstrip("/")
+    if backend_url:
+        yield {
+            "type": "status",
+            "message": f"Redaction backend for this turn: {backend_url}",
+        }
+
+    from eval.arize_monitoring import arize_session_context, langgraph_trace_config
+    from redaction_langgraph.graph import build_redaction_agent, graph_recursion_limit
+    from redaction_langgraph.llm_errors import (
+        is_context_overflow_error,
+        is_tool_call_json_parse_error,
+    )
+    from redaction_langgraph.message_context import (
+        get_trim_stats,
+        reset_trim_stats,
+        set_aggressive_trim,
+    )
+    from redaction_langgraph.workflow_continue import build_tool_call_json_retry_prompt
+
+    graph, system_message = build_redaction_agent(session_hash)
+    prior = get_messages(session_hash)
+    inputs = {"messages": [system_message, *prior, HumanMessage(content=prompt)]}
+    yield {"type": "agent_start"}
+
+    assistant_chunks: list[str] = []
+    stream_config = langgraph_trace_config(
+        session_hash, recursion_limit=graph_recursion_limit()
+    )
+
+    def _emit_stream(active_graph, active_inputs) -> list[dict]:
+        """Collect stream events; callers yield them and handle errors."""
+        out: list[dict] = []
+        reset_trim_stats()
+        compaction_noted = False
+        for event in active_graph.stream(
+            active_inputs, stream_mode="updates", config=stream_config
+        ):
+            stats = get_trim_stats()
+            if stats is not None and stats.trimmed and not compaction_noted:
+                compaction_noted = True
+                out.append(
+                    {
+                        "type": "status",
+                        "message": (
+                            f"Context compaction ({stats.tokens_before:,} → "
+                            f"{stats.tokens_after:,} tokens)."
+                        ),
+                    }
+                )
+            for node, update in event.items():
+                messages = update.get("messages") or []
+                for message in messages:
+                    if isinstance(message, AIMessage):
+                        text = stringify_message_content(message.content)
+                        if text:
+                            assistant_chunks.append(text)
+                        out.append(
+                            {
+                                "type": "message_update",
+                                "node": node,
+                                "role": "assistant",
+                                "content": text,
+                                "tool_calls": message.tool_calls or [],
+                            }
+                        )
+                    elif isinstance(message, ToolMessage):
+                        out.append(
+                            {
+                                "type": "message_update",
+                                "node": node,
+                                "role": "tool",
+                                "content": stringify_message_content(message.content),
+                                "tool_name": str(message.name or "tool"),
+                            }
+                        )
+                    else:
+                        content = getattr(message, "content", "")
+                        out.append(
+                            {
+                                "type": "message_update",
+                                "node": node,
+                                "role": getattr(message, "type", "unknown"),
+                                "content": content,
+                            }
+                        )
+        return out
+
+    try:
+        with arize_session_context(session_hash):
+            try:
+                for item in _emit_stream(graph, inputs):
+                    yield item
+            except Exception as exc:
+                if is_context_overflow_error(exc):
+                    assistant_chunks.clear()
+                    yield {
+                        "type": "status",
+                        "message": (
+                            "Prompt exceeded model context — retrying once with "
+                            "aggressive compaction…"
+                        ),
+                    }
+                    set_aggressive_trim(True)
+                    try:
+                        graph, _ = build_redaction_agent(
+                            session_hash, aggressive_compaction=True
+                        )
+                        for item in _emit_stream(graph, inputs):
+                            yield item
+                    finally:
+                        set_aggressive_trim(False)
+                elif is_tool_call_json_parse_error(exc):
+                    assistant_chunks.clear()
+                    yield {
+                        "type": "status",
+                        "message": (
+                            "Tool-call JSON was truncated or invalid — retrying once "
+                            "with a compact-script nudge…"
+                        ),
+                    }
+                    retry_inputs = {
+                        "messages": [
+                            *(inputs.get("messages") or []),
+                            HumanMessage(content=build_tool_call_json_retry_prompt()),
+                        ]
+                    }
+                    for item in _emit_stream(graph, retry_inputs):
+                        yield item
+                else:
+                    raise
+    except Exception as exc:
+        yield {"type": "error", "message": f"LangGraph agent failed: {exc}"}
+        return
+
+    append_turn(
+        session_hash,
+        user_text=prompt,
+        assistant_text="\n".join(assistant_chunks),
+    )
+    if request.get("sync_workspace_files"):
+        for item in collect_workspace_files_for_sync(session_hash):
+            yield {"type": "workspace_file", **item}
+    yield {"type": "agent_end", "message": "Agent finished."}

agent-redact/agentcore/package_runtime.py

@@ -0,0 +1,297 @@
+#!/usr/bin/env python3
+"""Package doc_redaction LangGraph agent code into an AgentCore app folder."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import shutil
+import stat
+import subprocess
+from pathlib import Path
+
+import tomllib
+
+_COPY_IGNORE = shutil.ignore_patterns(
+    "__pycache__", "*.pyc", ".pytest_cache", ".mypy_cache"
+)
+
+# Runtime Python deps to merge into the AgentCore app's pyproject.toml (not full pi-agent stack).
+RUNTIME_DEPENDENCIES: dict[str, str] = {
+    "gradio_client": ">=1.0.0",
+    "httpx": ">=0.28.0",
+    "python-dotenv": ">=1.0.0",
+    "langchain-openai": ">=1.0.0",
+    "langchain-core": ">=1.0.0",
+    "langgraph": ">=1.0.2",
+    "langchain-aws": ">=1.0.0",
+    "pymupdf": ">=1.24.0",
+    "pandas": ">=2.0.0",
+    "arize-otel": ">=0.9.0",
+    "arize-phoenix-otel": ">=0.16.0",
+    "openinference-instrumentation-langchain": ">=0.1.0",
+}
+
+MAIN_PY = '''"""doc_redaction LangGraph agent — packaged by agent-redact/agentcore/package_runtime.py."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_APP_ROOT = Path(__file__).resolve().parent
+_SHARED_DIR = _APP_ROOT / "shared"
+for path in (_APP_ROOT, _SHARED_DIR):
+    text = str(path)
+    if text not in sys.path:
+        sys.path.insert(0, text)
+
+from invoke_agent import bootstrap_runtime_env, invoke_redaction_agent  # noqa: E402
+
+bootstrap_runtime_env(_APP_ROOT)
+
+from bedrock_agentcore import BedrockAgentCoreApp  # noqa: E402
+
+app = BedrockAgentCoreApp()
+
+
+@app.entrypoint
+async def handler(request: dict):
+  async for event in invoke_redaction_agent(request):
+    yield event
+
+
+if __name__ == "__main__":
+    app.run()
+'''
+
+
+def _repo_root() -> Path:
+    return Path(__file__).resolve().parents[2]
+
+
+def _default_agentcore_app() -> Path:
+    return _repo_root() / "agent-redact" / "RedactionAgent" / "app" / "RedactionAgent"
+
+
+def _rmtree_robust(path: Path) -> None:
+    """Remove a directory tree on Windows / OneDrive (clears read-only files first)."""
+
+    def _on_rm_error(func, location, _exc_info) -> None:
+        os.chmod(location, stat.S_IWRITE)
+        func(location)
+
+    shutil.rmtree(path, onerror=_on_rm_error)
+
+
+def _copy_tree(src: Path, dest: Path, *, dry_run: bool) -> None:
+    if dry_run:
+        print(f"  copy tree {src} -> {dest}")
+        return
+    if dest.exists():
+        _rmtree_robust(dest)
+    shutil.copytree(src, dest, ignore=_COPY_IGNORE)
+
+
+def _copy_file(src: Path, dest: Path, *, dry_run: bool) -> None:
+    if dry_run:
+        print(f"  copy file {src} -> {dest}")
+        return
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(src, dest)
+
+
+def _replace_dependencies_block(text: str, deps: list[str]) -> str:
+    lines = text.splitlines()
+    out: list[str] = []
+    index = 0
+    while index < len(lines):
+        if lines[index].strip().startswith("dependencies"):
+            out.append("dependencies = [")
+            for dep in deps:
+                out.append(f'    "{dep}",')
+            out.append("]")
+            index += 1
+            while index < len(lines) and lines[index].strip() != "]":
+                index += 1
+            index += 1
+            continue
+        out.append(lines[index])
+        index += 1
+    return "\n".join(out) + ("\n" if text.endswith("\n") else "")
+
+
+def _merge_pyproject(pyproject_path: Path, *, dry_run: bool) -> None:
+    text = pyproject_path.read_text(encoding="utf-8")
+    if dry_run:
+        print(f"  merge deps into {pyproject_path}")
+        return
+    try:
+        data = tomllib.loads(text)
+    except tomllib.TOMLDecodeError as exc:
+        raise SystemExit(f"Could not parse {pyproject_path}: {exc}") from exc
+
+    existing: dict[str, str] = {}
+    for item in data.get("project", {}).get("dependencies", []):
+        if isinstance(item, str):
+            name = re.split(r"[<>=!~\[]", item, maxsplit=1)[0].strip()
+            existing[name.lower()] = item
+
+    for name, spec in RUNTIME_DEPENDENCIES.items():
+        key = name.lower()
+        if key not in existing:
+            existing[key] = f"{name}{spec}"
+
+    merged = [existing[k] for k in sorted(existing, key=str.lower)]
+    pyproject_path.write_text(
+        _replace_dependencies_block(text, merged), encoding="utf-8"
+    )
+
+
+def package_runtime(
+    target: Path,
+    *,
+    dry_run: bool = False,
+) -> list[str]:
+    """Sync monorepo redaction agent sources into *target* (AgentCore app folder)."""
+    repo = _repo_root()
+    agent_redact = repo / "agent-redact"
+    agentcore = agent_redact / "agentcore"
+    actions: list[str] = []
+
+    def log(msg: str) -> None:
+        actions.append(msg)
+        print(msg)
+
+    log(f"Packaging doc_redaction runtime -> {target}")
+
+    _copy_tree(
+        agent_redact / "redaction_langgraph",
+        target / "redaction_langgraph",
+        dry_run=dry_run,
+    )
+
+    _copy_tree(
+        agent_redact / "eval",
+        target / "eval",
+        dry_run=dry_run,
+    )
+
+    shared_dest = target / "shared"
+    for name in ("remote_redaction.py",):
+        _copy_file(agent_redact / "shared" / name, shared_dest / name, dry_run=dry_run)
+
+    _copy_file(
+        agentcore / "bundle_support" / "session_workspace.py",
+        shared_dest / "session_workspace.py",
+        dry_run=dry_run,
+    )
+
+    for module in ("invoke_agent.py", "session_store.py", "workspace_sync.py"):
+        _copy_file(agentcore / module, target / module, dry_run=dry_run)
+
+    if dry_run:
+        log(f"  write {target / 'main.py'}")
+    else:
+        (target / "main.py").write_text(MAIN_PY, encoding="utf-8")
+        log(f"wrote {target / 'main.py'}")
+
+    pyproject = target / "pyproject.toml"
+    if pyproject.is_file():
+        _merge_pyproject(pyproject, dry_run=dry_run)
+        log(f"merged runtime dependencies into {pyproject}")
+    elif dry_run:
+        log(f"  skip pyproject merge (no {pyproject} — run agentcore create first)")
+    else:
+        raise SystemExit(f"Missing {pyproject} — run agentcore create first.")
+
+    env_example = target / "agentcore.env.example"
+    env_local = target / "agentcore.env"
+    example_text = """# Loaded at runtime startup when present in the CodeZip (see invoke_agent.bootstrap_runtime_env).
+# Also set these on the AgentCore runtime in AWS if you prefer console/config-bundle env.
+# CDK + AgentCore: use main Express HTTPS (ExpressServiceEndpoint), not Service Connect.
+DOC_REDACTION_GRADIO_URL=https://your-doc-redaction-host.example
+AGENT_DEFAULT_PROVIDER=amazon-bedrock
+AGENT_DEFAULT_MODEL=anthropic.claude-sonnet-4-6
+AWS_REGION=eu-west-2
+AGENT_WORKSPACE_DIR=/tmp/agentcore-workspace
+AGENT_DEFAULT_OCR_METHOD=paddle
+AGENT_DEFAULT_PII_METHOD=Local
+# Optional Arize AX / Phoenix tracing for in-process LangGraph
+# (see agent-redact/eval/arize_monitoring.py).
+# ARIZE_TRACING_ENABLED=true
+# ARIZE_BACKEND=ax
+# ARIZE_BACKEND=phoenix
+# PHOENIX_COLLECTOR_ENDPOINT=http://localhost:6006
+# ARIZE_SPACE_ID=
+# ARIZE_API_KEY=
+# ARIZE_PROJECT_NAME=doc-redaction-langgraph
+# ARIZE_ENDPOINT=europe
+"""
+    if dry_run:
+        log(f"  write {env_example}")
+        if env_local.is_file():
+            log(f"  keep existing {env_local}")
+    else:
+        env_example.write_text(example_text, encoding="utf-8")
+        log(f"wrote {env_example}")
+        if not env_local.is_file():
+            env_local.write_text(example_text, encoding="utf-8")
+            log(f"wrote {env_local} (copy from example — edit before deploy)")
+
+    return actions
+
+
+def run_deploy(agentcore_project: Path) -> None:
+    env = dict(**{k: v for k, v in __import__("os").environ.items()})
+    env.setdefault("UV_LINK_MODE", "copy")
+    subprocess.run(
+        ["agentcore", "deploy"],
+        cwd=str(agentcore_project),
+        check=True,
+        env=env,
+    )
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Package doc_redaction LangGraph agent into an AgentCore app folder.",
+    )
+    parser.add_argument(
+        "--target",
+        type=Path,
+        default=_default_agentcore_app(),
+        help="AgentCore app folder (default: agent-redact/RedactionAgent/app/RedactionAgent)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Print actions without writing files",
+    )
+    parser.add_argument(
+        "--deploy",
+        action="store_true",
+        help="Run agentcore deploy from the RedactionAgent project after packaging",
+    )
+    args = parser.parse_args(argv)
+
+    target = args.target.resolve()
+    package_runtime(target, dry_run=args.dry_run)
+
+    if args.deploy:
+        if args.dry_run:
+            print("Skipping deploy (--dry-run).")
+            return 0
+        project = target.parent.parent
+        if not (project / "agentcore" / "agentcore.json").is_file():
+            raise SystemExit(f"Not an AgentCore project: {project}")
+        print(f"Running agentcore deploy in {project} ...")
+        run_deploy(project)
+
+    print("Done.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agent-redact/agentcore/session_store.py

@@ -0,0 +1,60 @@
+"""In-process conversation history for AgentCore / LangGraph orchestration."""
+
+from __future__ import annotations
+
+import threading
+from typing import Any
+
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
+
+_lock = threading.Lock()
+_sessions: dict[str, list[BaseMessage]] = {}
+
+
+def _session_key(session_hash: str | None) -> str:
+    key = (session_hash or "").strip()
+    return key or "default"
+
+
+def get_messages(session_hash: str | None) -> list[BaseMessage]:
+    """Return a copy of stored messages for *session_hash*."""
+    key = _session_key(session_hash)
+    with _lock:
+        return list(_sessions.get(key, []))
+
+
+def clear_session(session_hash: str | None) -> None:
+    """Drop conversation history for *session_hash*."""
+    key = _session_key(session_hash)
+    with _lock:
+        _sessions.pop(key, None)
+
+
+def append_turn(
+    session_hash: str | None,
+    *,
+    user_text: str,
+    assistant_text: str = "",
+) -> None:
+    """Append one user turn and optional assistant reply."""
+    key = _session_key(session_hash)
+    with _lock:
+        history = _sessions.setdefault(key, [])
+        history.append(HumanMessage(content=user_text))
+        if assistant_text.strip():
+            history.append(AIMessage(content=assistant_text.strip()))
+
+
+def stringify_message_content(content: Any) -> str:
+    """Normalize LangChain message content to plain text."""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: list[str] = []
+        for block in content:
+            if isinstance(block, str):
+                parts.append(block)
+            elif isinstance(block, dict) and block.get("type") == "text":
+                parts.append(str(block.get("text") or ""))
+        return "".join(parts)
+    return str(content or "")

agent-redact/agentcore/workspace_sync.py

@@ -0,0 +1,83 @@
+"""Sync session workspace files to/from AgentCore invoke payloads."""
+
+from __future__ import annotations
+
+import base64
+import os
+from pathlib import Path
+
+_DEFAULT_MAX_BYTES = 8 * 1024 * 1024
+
+
+def max_workspace_sync_bytes() -> int:
+    raw = (os.environ.get("AGENTCORE_MAX_UPLOAD_BYTES") or "").strip()
+    if raw.isdigit():
+        return int(raw)
+    return _DEFAULT_MAX_BYTES
+
+
+def _ensure_session_workspace(session_hash: str | None) -> Path:
+    try:
+        from bundle_support.session_workspace import ensure_session_workspace
+    except ImportError:
+        from session_workspace import (
+            ensure_session_workspace,  # type: ignore[no-redef]
+        )
+
+    return ensure_session_workspace(session_hash or "")
+
+
+def apply_workspace_files(session_hash: str | None, files: list[dict]) -> list[str]:
+    """Write base64-encoded files into the session workspace. Returns relative paths written."""
+    if not files:
+        return []
+    root = _ensure_session_workspace(session_hash).resolve()
+    written: list[str] = []
+    for item in files:
+        if not isinstance(item, dict):
+            continue
+        relative = str(item.get("relative_path") or item.get("name") or "").strip()
+        encoded = str(item.get("content_base64") or "").strip()
+        if not relative or not encoded:
+            continue
+        dest = (root / relative).resolve()
+        try:
+            dest.relative_to(root)
+        except ValueError:
+            continue
+        payload = base64.b64decode(encoded, validate=True)
+        if len(payload) > max_workspace_sync_bytes():
+            continue
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_bytes(payload)
+        written.append(str(dest.relative_to(root)).replace("\\", "/"))
+    return written
+
+
+def collect_workspace_files_for_sync(
+    session_hash: str | None,
+    *,
+    prefix: str = "redact/",
+) -> list[dict[str, str]]:
+    """Collect workspace files under *prefix* for download to the Gradio client."""
+    root = _ensure_session_workspace(session_hash).resolve()
+    if not root.is_dir():
+        return []
+    limit = max_workspace_sync_bytes()
+    out: list[dict[str, str]] = []
+    for path in sorted(root.rglob("*")):
+        if not path.is_file():
+            continue
+        rel = str(path.relative_to(root)).replace("\\", "/")
+        if prefix and not rel.startswith(prefix.lstrip("/")):
+            continue
+        size = path.stat().st_size
+        if size > limit:
+            continue
+        out.append(
+            {
+                "relative_path": rel,
+                "content_base64": base64.b64encode(path.read_bytes()).decode("ascii"),
+            }
+        )
+    return out

agent-redact/eval/__init__.py

@@ -0,0 +1 @@
+"""Agent evaluation and observability helpers (e.g. Arize AX / Phoenix tracing)."""

agent-redact/eval/arize_monitoring.py

@@ -0,0 +1,222 @@
+"""Arize / Phoenix OpenTelemetry tracing for LangGraph agent runs.
+
+Call :func:`setup_arize_ax_tracing` once **before** any ``langchain`` /
+``langchain_core`` import so ``LangChainInstrumentor`` can patch the stack.
+
+Multi-turn chats: wrap each ``graph.stream`` with :func:`arize_session_context`
+and pass :func:`langgraph_trace_config` so turns share ``session.id`` (Gradio
+``session_hash``) and appear under one Sessions row in Arize AX or Phoenix.
+
+Environment (see ``config/agent.env.example``):
+
+- ``ARIZE_TRACING_ENABLED`` — set ``true`` / ``1`` / ``yes`` / ``on`` to enable
+- ``ARIZE_BACKEND`` — ``ax`` (default, hosted Arize AX) or ``phoenix`` (local /
+  self-hosted Phoenix)
+- ``ARIZE_PROJECT_NAME`` — default ``doc-redaction-langgraph`` (Phoenix also
+  accepts ``PHOENIX_PROJECT_NAME``)
+- AX (``ARIZE_BACKEND=ax``): ``ARIZE_SPACE_ID`` / ``ARIZE_API_KEY`` required;
+  ``ARIZE_ENDPOINT`` — ``europe`` (default) or ``us``
+- Phoenix (``ARIZE_BACKEND=phoenix``): ``PHOENIX_COLLECTOR_ENDPOINT`` (default
+  ``http://localhost:6006``); optional ``PHOENIX_API_KEY``
+
+Smoke check
+-----------
+1. Set ``AGENT_ORCHESTRATOR=langgraph`` and tracing env vars.
+2. For Phoenix: run ``phoenix serve``, set ``ARIZE_BACKEND=phoenix``, then
+   start the agent Gradio app and send one short chat that triggers an LLM call.
+3. Confirm spans in Phoenix (``http://localhost:6006``) or the Arize AX project.
+4. Send a follow-up in the same Gradio session; both turns should share one
+   Sessions row (``session.id`` = Gradio ``session_hash``).
+5. With tracing disabled or ``AGENT_ORCHESTRATOR=pi``: confirm no errors and
+   no collector traffic.
+"""
+
+from __future__ import annotations
+
+import os
+import warnings
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import Any
+
+_INITIALIZED = False
+
+
+def _env_truthy(name: str) -> bool:
+    return (os.environ.get(name) or "").strip().lower() in {
+        "1",
+        "true",
+        "yes",
+        "on",
+    }
+
+
+def _tracing_backend() -> str:
+    """Return ``ax`` or ``phoenix`` from ``ARIZE_BACKEND`` (default ``ax``)."""
+    raw = (os.environ.get("ARIZE_BACKEND") or "ax").strip().lower()
+    if raw in {"phoenix", "local", "oss"}:
+        return "phoenix"
+    return "ax"
+
+
+def _project_name() -> str:
+    return (
+        os.environ.get("PHOENIX_PROJECT_NAME")
+        or os.environ.get("ARIZE_PROJECT_NAME")
+        or "doc-redaction-langgraph"
+    ).strip()
+
+
+def _phoenix_otlp_endpoint(collector: str) -> str:
+    """Normalize collector base URL to an HTTP OTLP traces endpoint when needed."""
+    ep = collector.strip().rstrip("/")
+    if not ep:
+        ep = "http://localhost:6006"
+    if ep.endswith("/v1/traces") or ep.endswith(":4317"):
+        return ep
+    return f"{ep}/v1/traces"
+
+
+def arize_session_id(session_hash: str | None) -> str | None:
+    """Return a non-empty session id for OpenInference, or None."""
+    sid = (session_hash or "").strip()
+    return sid or None
+
+
+def langgraph_trace_config(
+    session_hash: str | None,
+    *,
+    recursion_limit: int,
+) -> dict[str, Any]:
+    """LangGraph ``config`` with recursion limit and Arize/LangChain session keys."""
+    config: dict[str, Any] = {"recursion_limit": recursion_limit}
+    sid = arize_session_id(session_hash)
+    if sid:
+        # LangChainInstrumentor groups turns via any of these metadata keys.
+        config["metadata"] = {
+            "session_id": sid,
+            "thread_id": sid,
+            "conversation_id": sid,
+        }
+        config["configurable"] = {"thread_id": sid}
+    return config
+
+
+@contextmanager
+def arize_session_context(session_hash: str | None) -> Iterator[None]:
+    """Attach OpenInference ``session.id`` for the duration of a LangGraph turn.
+
+    No-op when *session_hash* is empty or ``openinference`` is not installed.
+    Safe to use inside generators (context stays active across ``yield``).
+    """
+    sid = arize_session_id(session_hash)
+    if not sid:
+        yield
+        return
+    try:
+        from openinference.instrumentation import using_session
+    except ImportError:
+        yield
+        return
+    with using_session(session_id=sid):
+        yield
+
+
+def _setup_phoenix_tracing(project_name: str) -> bool:
+    """Register Phoenix OTEL exporter + LangChainInstrumentor."""
+    try:
+        from openinference.instrumentation.langchain import LangChainInstrumentor
+        from phoenix.otel import register
+    except ImportError as exc:
+        warnings.warn(
+            f"Phoenix tracing dependencies unavailable ({exc}); tracing skipped. "
+            "Install arize-phoenix-otel and openinference-instrumentation-langchain.",
+            UserWarning,
+            stacklevel=3,
+        )
+        return False
+
+    collector = (
+        os.environ.get("PHOENIX_COLLECTOR_ENDPOINT") or "http://localhost:6006"
+    ).strip()
+    endpoint = _phoenix_otlp_endpoint(collector)
+    register_kwargs: dict[str, Any] = {
+        "project_name": project_name,
+        "endpoint": endpoint,
+    }
+    api_key = (os.environ.get("PHOENIX_API_KEY") or "").strip()
+    if api_key:
+        register_kwargs["api_key"] = api_key
+
+    tracer_provider = register(**register_kwargs)
+    LangChainInstrumentor().instrument(tracer_provider=tracer_provider)
+    return True
+
+
+def _setup_ax_tracing(project_name: str) -> bool:
+    """Register Arize AX tracer + LangChainInstrumentor."""
+    space_id = (os.environ.get("ARIZE_SPACE_ID") or "").strip()
+    api_key = (os.environ.get("ARIZE_API_KEY") or "").strip()
+    if not space_id or not api_key:
+        warnings.warn(
+            "ARIZE_TRACING_ENABLED is set but ARIZE_SPACE_ID / ARIZE_API_KEY "
+            "are missing; Arize AX tracing is skipped.",
+            UserWarning,
+            stacklevel=3,
+        )
+        return False
+
+    try:
+        from arize.otel import Endpoint, register
+        from openinference.instrumentation.langchain import LangChainInstrumentor
+    except ImportError as exc:
+        warnings.warn(
+            f"Arize AX tracing dependencies unavailable ({exc}); tracing skipped. "
+            "Install arize-otel and openinference-instrumentation-langchain.",
+            UserWarning,
+            stacklevel=3,
+        )
+        return False
+
+    endpoint_raw = (os.environ.get("ARIZE_ENDPOINT") or "europe").strip().lower()
+    if endpoint_raw in {"us", "arize", "default"}:
+        endpoint = Endpoint.ARIZE
+    else:
+        # Default to Europe (draft / Lambeth deployment).
+        endpoint = Endpoint.ARIZE_EUROPE
+
+    tracer_provider = register(
+        space_id=space_id,
+        api_key=api_key,
+        project_name=project_name,
+        endpoint=endpoint,
+    )
+    LangChainInstrumentor().instrument(tracer_provider=tracer_provider)
+    return True
+
+
+def setup_arize_ax_tracing() -> bool:
+    """Register Arize AX or Phoenix tracer + LangChainInstrumentor.
+
+    Backend is selected via ``ARIZE_BACKEND`` (``ax`` default, or ``phoenix``).
+
+    Returns True if tracing was enabled (or already initialized). Soft-fails
+    with a warning when partially configured or packages are missing.
+    """
+    global _INITIALIZED
+    if _INITIALIZED:
+        return True
+
+    if not _env_truthy("ARIZE_TRACING_ENABLED"):
+        return False
+
+    project_name = _project_name()
+    backend = _tracing_backend()
+    if backend == "phoenix":
+        ok = _setup_phoenix_tracing(project_name)
+    else:
+        ok = _setup_ax_tracing(project_name)
+
+    if ok:
+        _INITIALIZED = True
+    return ok

agent-redact/pi-agent/.dockerignore

@@ -0,0 +1,11 @@
+.git
+.github
+**/__pycache__
+**/*.pyc
+**/.pytest_cache
+**/node_modules
+workspace
+output
+input
+config/agent.env
+config/pi_agent.env

agent-redact/pi-agent/.gitattributes

@@ -0,0 +1,2 @@
+# Example PDFs must be plain files in the Space repo (not Git LFS pointers).
+*.pdf -filter -diff -merge

agent-redact/pi-agent/Dockerfile

@@ -0,0 +1,179 @@
+# syntax=docker/dockerfile:1
+# Pi agent image (dev + production). Build from monorepo root:
+#   docker build -f agent-redact/pi-agent/Dockerfile --target dev .
+#   docker build -f agent-redact/pi-agent/Dockerfile --target runtime .
+# Root .dockerignore must allow config/*.example into the context (secrets stay gitignored).
+#
+# Targets:
+#   dev     — docker-compose: Pi CLI + Python deps; app tree bind-mounted at runtime.
+#   runtime — HF Space / AWS ECS: baked agent-redact tree, non-root user, named volumes.
+
+# ===================================================================
+# Stage 1: Pi CLI (Node) — isolated so the runtime base stays Python 3.12
+# ===================================================================
+FROM public.ecr.aws/docker/library/node:24.16.0-slim AS pi-cli
+
+ENV NPM_CONFIG_PREFIX=/opt/pi
+ENV PATH="/opt/pi/bin:${PATH}"
+
+RUN npm install -g --ignore-scripts @earendil-works/pi-coding-agent
+
+# ===================================================================
+# Stage 2: Shared Python base (aligned with main app Dockerfile)
+# ===================================================================
+FROM public.ecr.aws/docker/library/python:3.12.13-slim-trixie AS pi-base
+
+ENV NODE_ENV=production
+ENV DEBIAN_FRONTEND=noninteractive
+ENV NPM_CONFIG_LOGLEVEL=warn
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV APP_HOME=/home/user
+ENV AGENT_WORKDIR=/workspace/doc_redaction
+ENV PYTHONPATH=${AGENT_WORKDIR}:${AGENT_WORKDIR}/agent-redact:${AGENT_WORKDIR}/agent-redact/shared:${AGENT_WORKDIR}/agent-redact/pi:${AGENT_WORKDIR}/agent-redact/agentcore
+ENV GRADIO_SERVER_NAME=0.0.0.0
+ENV MPLCONFIGDIR=/tmp/matplotlib_cache/
+ENV XDG_CACHE_HOME=/tmp/xdg_cache/user_1000
+ENV PATH="/opt/pi/bin:${PATH}"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash \
+    git \
+    curl \
+    ca-certificates \
+    procps \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+COPY --from=pi-cli /opt/pi /opt/pi
+COPY --from=pi-cli /usr/local/bin/node /usr/local/bin/node
+
+COPY agent-redact/requirements_agent.txt /tmp/requirements_agent.txt
+RUN pip install --no-cache-dir -r /tmp/requirements_agent.txt \
+    && rm /tmp/requirements_agent.txt
+
+# ===================================================================
+# Stage 3: Dev — thin image for docker-compose (repo bind-mounted)
+# ===================================================================
+FROM pi-base AS dev
+
+ENV HOME=${APP_HOME}
+ENV AGENT_WORKSPACE_DIR=${APP_HOME}/app/workspace
+ENV AGENT_UPLOAD_ROOT=/tmp/gradio
+ENV AGENT_SESSION_DIR=${APP_HOME}/.pi/agent/sessions
+
+RUN useradd -m -u 1000 user \
+    && mkdir -p \
+        ${APP_HOME}/app/workspace \
+        ${APP_HOME}/.pi/agent/sessions \
+        ${AGENT_WORKDIR} \
+        /tmp/gradio \
+        /tmp/matplotlib_cache \
+        ${XDG_CACHE_HOME} \
+    && chown -R user:user ${APP_HOME} ${AGENT_WORKDIR} \
+    && chown user:user /tmp/gradio /tmp/matplotlib_cache ${XDG_CACHE_HOME} \
+    && chmod 1777 /tmp/gradio /tmp/matplotlib_cache \
+    && chmod 700 ${XDG_CACHE_HOME}
+
+WORKDIR ${AGENT_WORKDIR}
+
+USER user
+
+RUN pi --version
+
+# Compose overrides entrypoint with agent-redact/shared/start.sh on the bind mount.
+
+# ===================================================================
+# Stage 4: Runtime — baked app for Hugging Face Space and AWS ECS
+# ===================================================================
+FROM pi-base AS runtime
+
+ENV AGENT_DEPLOYMENT_PROFILE=hf-space
+ENV AGENT_DEFAULT_PROVIDER=google-gemini
+ENV AGENT_DEFAULT_MODEL=gemini-flash-lite-latest
+ENV DOC_REDACTION_GRADIO_URL=https://seanpedrickcase-document-redaction.hf.space
+ENV HOME=${APP_HOME}
+ENV AGENT_WORKDIR=/workspace/doc_redaction
+# Fargate uses volume mounts under ${APP_HOME}/app/workspace (CDK chown entrypoint).
+# ECS Express has no mounts — CDK sets AGENT_WORKSPACE_DIR=/tmp/agent-workspace at deploy.
+ENV AGENT_WORKSPACE_DIR=${APP_HOME}/app/workspace
+ENV AGENT_UPLOAD_ROOT=/tmp/gradio
+ENV AGENT_SESSION_DIR=/tmp/agent-sessions
+ENV AGENT_CODING_AGENT_DIR=/tmp/agent-coding
+ENV PI_CODING_AGENT_DIR=/tmp/agent-coding
+ENV PI_CODING_AGENT_SESSION_DIR=/tmp/agent-sessions
+ENV ACCESS_LOGS_FOLDER=/tmp/agent-logs/
+ENV USAGE_LOGS_FOLDER=/tmp/agent-usage/
+ENV FEEDBACK_LOGS_FOLDER=/tmp/agent-feedback/
+ENV PI_OFFLINE=1
+ENV PI_SKIP_VERSION_CHECK=1
+ENV AGENT_GRADIO_SHOW_EXAMPLES=true
+ENV AGENT_UI_HOST=0.0.0.0
+ENV AGENT_UI_PORT=7860
+ENV AGENT_GRADIO_PORT=7860
+ENV GRADIO_SERVER_NAME=0.0.0.0
+ENV GRADIO_SERVER_PORT=7860
+ENV GRADIO_ANALYTICS_ENABLED=False
+ENV RUN_FASTAPI=False
+
+WORKDIR ${AGENT_WORKDIR}
+
+COPY agent-redact/shared agent-redact/shared
+COPY agent-redact/pi agent-redact/pi
+COPY skills skills
+COPY tools tools
+# Committed template only (see sync-manifest.txt); runtime secrets come from S3/env on ECS.
+COPY config/agent.env.example config/agent.env.example
+COPY intros intros
+COPY AGENTS.md AGENTS.md
+COPY doc_redaction/example_data doc_redaction/example_data
+
+RUN test -f doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf \
+    && test -f doc_redaction/example_data/graduate-job-example-cover-letter.pdf \
+    && ! head -1 doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf \
+        | grep -q "^version https://git-lfs.github.com/spec/v1"
+
+RUN useradd -m -u 1000 user \
+    && mkdir -p \
+        ${APP_HOME}/app/workspace \
+        ${APP_HOME}/.pi/agent \
+        /tmp/gradio \
+        /tmp/agent-sessions \
+        /tmp/matplotlib_cache \
+        ${XDG_CACHE_HOME} \
+    && chown user:user \
+        ${APP_HOME}/app/workspace \
+        ${APP_HOME}/.pi \
+        /tmp/gradio \
+        /tmp/agent-sessions \
+        /tmp/matplotlib_cache \
+        ${XDG_CACHE_HOME} \
+    && chmod 755 ${APP_HOME}/app/workspace ${APP_HOME}/.pi \
+    && chmod 1777 /tmp/gradio /tmp/agent-sessions /tmp/matplotlib_cache \
+    && chmod 700 ${XDG_CACHE_HOME} \
+    && chown -R root:root ${AGENT_WORKDIR} \
+    && find ${AGENT_WORKDIR} -type d -exec chmod 755 {} \; \
+    && find ${AGENT_WORKDIR} -type f -exec chmod 644 {} \; \
+    && mkdir -p ${APP_HOME}/app \
+    && chown user:user ${APP_HOME}/app
+
+COPY agent-redact/pi-agent/entrypoint-ecs.sh /usr/local/bin/entrypoint-ecs.sh
+COPY agent-redact/pi-agent/entrypoint.sh ${APP_HOME}/app/entrypoint.sh
+RUN sed -i 's/\r$//' /usr/local/bin/entrypoint-ecs.sh ${APP_HOME}/app/entrypoint.sh \
+    && chmod +x /usr/local/bin/entrypoint-ecs.sh ${APP_HOME}/app/entrypoint.sh
+
+# Writable paths only via runtime mounts (read-only root FS friendly).
+VOLUME ["${APP_HOME}/app/workspace"]
+VOLUME ["/tmp/gradio"]
+VOLUME ["/tmp/agent-sessions"]
+VOLUME ["/tmp/matplotlib_cache"]
+VOLUME ["${XDG_CACHE_HOME}"]
+VOLUME ["/tmp"]
+VOLUME ["/var/tmp"]
+
+USER user
+
+RUN pi --version
+
+EXPOSE 7860
+
+ENTRYPOINT ["/home/user/app/entrypoint.sh"]

agent-redact/pi-agent/README.md

@@ -0,0 +1,46 @@
+---
+title: Agentic Document Redaction
+emoji: 🤖
+colorFrom: blue
+colorTo: yellow
+sdk: docker
+app_file: agent-redact/shared/gradio_app.py
+pinned: false
+license: agpl-3.0
+short_description: Agentic interface to redact PDF documents
+---
+
+# Pi agent — agentic document redaction
+
+Orchestrate document redaction with **[Pi](https://github.com/earendil-works/pi)** and **Google Gemini**. Heavy redaction runs on a separate **private [doc_redaction](https://huggingface.co/spaces/seanpedrickcase/document_redaction)** Hugging Face Space (simple text extraction + Local PII).
+
+## Before you start
+
+1. **Gemini API key** — paste in **Agent backend** → **Apply backend** (session-only; not stored on disk).
+2. **HF token** — Space admin should set `HF_TOKEN` under **Settings → Secrets** so this Space can call the private redaction backend. Users may optionally override per session in the UI.
+
+## Limitations
+
+- **No face or signature VLM** — text-layer PII only via Local spaCy/Presidio on the remote Space.
+- **No Pass 2 VLM** on this deployment.
+- **Ephemeral storage** — download deliverables from **Workspace output files** before the Space restarts.
+- **Human review** — outputs are not guaranteed complete; review redacted PDFs before release.
+
+## Defaults
+
+| Setting | Value |
+|---------|--------|
+| Pi LLM | Gemini (`gemini-flash-latest` default) |
+| Redaction backend | `https://seanpedrickcase-document-redaction.hf.space` |
+| Text extraction | `Local model - selectable text` |
+| PII detection | `Local` |
+
+## Examples
+
+Two sample PDFs load in **Redaction task** → **Try an example** (same demos as the main doc_redaction app). Examples are **on by default**; set Space variable `AGENT_GRADIO_SHOW_EXAMPLES=false` to hide them. (`SHOW_PI_EXAMPLES` is also accepted.)
+
+If examples do not appear, the UI shows a short status message (usually missing PDFs in the image — rebuild after a successful sync with LFS materialization).
+
+## Development
+
+This Space is synced from the [doc_redaction monorepo](https://github.com/seanpedrick-case/doc_redaction) on pushes to **`dev`** (see `.github/workflows/sync-pi-agent-space.yml`). Space: [seanpedrickcase/agentic_document_redaction](https://huggingface.co/spaces/seanpedrickcase/agentic_document_redaction).

agent-redact/pi-agent/entrypoint-ecs.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# ECS Fargate: ephemeral volume mounts are root-owned; chown then drop to user (image USER).
+set -euo pipefail
+
+for dir in /tmp/agent-coding /tmp/agent-logs /tmp/agent-usage /tmp/agent-feedback \
+    /home/user/app/workspace /tmp/gradio /tmp/agent-sessions; do
+  mkdir -p "$dir"
+  chown -R user:user "$dir"
+done
+
+cd /workspace/doc_redaction
+exec su -s /bin/bash user -c "$*"

agent-redact/pi-agent/entrypoint.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+set -e
+
+echo "Starting Pi agent (profile=${AGENT_DEPLOYMENT_PROFILE:-unknown})"
+
+for dir in \
+    "${AGENT_CODING_AGENT_DIR:-/tmp/agent-coding}" \
+    "${AGENT_WORKSPACE_DIR:-/home/user/app/workspace}" \
+    "${AGENT_UPLOAD_ROOT:-/tmp/gradio}" \
+    "${AGENT_SESSION_DIR:-/tmp/agent-sessions}" \
+    "${ACCESS_LOGS_FOLDER:-/tmp/agent-logs}" \
+    "${USAGE_LOGS_FOLDER:-/tmp/agent-usage}" \
+    "${FEEDBACK_LOGS_FOLDER:-/tmp/agent-feedback}" \
+    "${MPLCONFIGDIR:-/tmp/matplotlib_cache}" \
+    "${XDG_CACHE_HOME:-/tmp/xdg_cache/user_1000}"; do
+    mkdir -p "$dir" 2>/dev/null || true
+    if [ ! -w "$dir" ]; then
+        echo "WARNING: Directory $dir is not writable by current user (uid=$(id -u)). File I/O may fail." >&2
+    fi
+done
+
+cd "${AGENT_WORKDIR:-/workspace/doc_redaction}"
+
+echo "Entrypoint environment: AGENT_WORKSPACE_DIR=${AGENT_WORKSPACE_DIR:-} AGENT_UI_HOST=${AGENT_UI_HOST:-} AGENT_UI_PORT=${AGENT_UI_PORT:-} AGENT_GRADIO_PORT=${AGENT_GRADIO_PORT:-} GRADIO_SERVER_NAME=${GRADIO_SERVER_NAME:-} GRADIO_SERVER_PORT=${GRADIO_SERVER_PORT:-} RUN_FASTAPI=${RUN_FASTAPI:-}"
+
+python3 agent-redact/pi/pi_agent_config.py
+if [ "${RUN_FASTAPI:-False}" = "True" ]; then
+  exec uvicorn gradio_app:app \
+    --app-dir agent-redact/shared \
+    --host "${GRADIO_SERVER_NAME:-0.0.0.0}" \
+    --port "${AGENT_GRADIO_PORT:-${GRADIO_SERVER_PORT:-7860}}" \
+    --proxy-headers \
+    --forwarded-allow-ips "*"
+else
+  exec python3 agent-redact/shared/gradio_app.py
+fi

agent-redact/pi-agent/sync-manifest.txt

@@ -0,0 +1,13 @@
+# Paths copied from the monorepo root into the flattened Pi agent HF Space repo.
+agent-redact/requirements_agent.txt
+agent-redact/shared
+agent-redact/pi
+agent-redact/pi-agent/entrypoint.sh
+agent-redact/pi-agent/entrypoint-ecs.sh
+skills
+tools
+config/agent.env.example
+intros/pi_intro.txt
+AGENTS.md
+doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf
+doc_redaction/example_data/graduate-job-example-cover-letter.pdf

agent-redact/pi-agent/sync_to_space.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Flatten monorepo paths into a temp directory for the Pi agent HF Space repo.
+# Usage (from repo root):
+#   agent-redact/pi-agent/sync_to_space.sh /path/to/output-dir
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+OUT="${1:?Output directory required}"
+MANIFEST="$(dirname "$0")/sync-manifest.txt"
+
+_is_lfs_pointer() {
+  [[ -f "$1" ]] && head -1 "$1" 2>/dev/null | grep -q "^version https://git-lfs.github.com/spec/v1"
+}
+
+rm -rf "$OUT"
+mkdir -p "$OUT"
+
+cp "$(dirname "$0")/Dockerfile" "$OUT/Dockerfile"
+cp "$(dirname "$0")/README.md" "$OUT/README.md"
+cp "$(dirname "$0")/.dockerignore" "$OUT/.dockerignore"
+cp "$(dirname "$0")/.gitattributes" "$OUT/.gitattributes"
+
+while IFS= read -r line || [[ -n "$line" ]]; do
+  line="${line%%#*}"
+  line="$(echo "$line" | xargs)"
+  [[ -z "$line" ]] && continue
+  src="$ROOT/$line"
+  if [[ ! -e "$src" ]]; then
+    echo "Missing: $src" >&2
+    exit 1
+  fi
+  dest="$OUT/$line"
+  mkdir -p "$(dirname "$dest")"
+  cp -a "$src" "$dest"
+  if [[ "$line" == *.pdf ]] && _is_lfs_pointer "$dest"; then
+    echo "Copied file is a Git LFS pointer, not a PDF: $line" >&2
+    echo "Run 'git lfs pull' in the monorepo before syncing." >&2
+    exit 1
+  fi
+done < "$MANIFEST"
+
+echo "Flattened Pi agent Space tree: $OUT"

agent-redact/pi/agent/README.md

@@ -0,0 +1,223 @@
+# Pi agent config (Docker)
+
+Runtime Pi config is **generated at container start** by [`agent-redact/pi/pi_agent_config.py`](../pi_agent_config.py) into `~/.pi/agent/models.json` and `~/.pi/agent/settings.json`.
+
+Files in this folder (`settings.json`, `models.json`) are **templates/references** only — they are no longer bind-mounted into the container.
+
+## LLM backends (Pi orchestration)
+
+The Pi agent (chat + redaction orchestration) can use:
+
+| Provider key | Label | Pi API | Auth |
+|--------------|-------|--------|------|
+| `llama-cpp` | Local (llama-cpp) | `openai-completions` | None (local llama-inference) |
+| `google-gemini` | Gemini | `google-generative-ai` | `GEMINI_API_KEY` or `GOOGLE_API_KEY` |
+| `amazon-bedrock` | AWS Bedrock | `bedrock-converse-stream` | AWS SDK credentials (`AWS_ACCESS_KEY_ID`, etc.) |
+
+This is separate from doc_redaction **Pass 2 VLM** (`{VLM_BASE_URL}` in redaction prompts), which still targets local llama-inference by default.
+
+## Orchestration backends (`AGENT_ORCHESTRATOR`)
+
+The Gradio UI can drive four orchestration backends (see [`agent_runtime.py`](../agent_runtime.py)):
+
+| Value | Runtime | Notes |
+|-------|---------|-------|
+| `pi` (default) | Pi coding agent (`pi --mode rpc`) | Full bash + skills; retained for HF Space and gradual migration |
+| `langgraph` | LangGraph ReAct agent | Curated Python tools only (no shell); local llama.cpp / Bedrock / Gemini |
+| `agentcore` | Bedrock AgentCore **Runtime** | LangGraph bundle via `AGENTCORE_RUNTIME_URL`; see **[AgentCore install guide](../../agentcore/README.md)** |
+| `agentcore-harness` (alias `harness`) | Bedrock AgentCore **Harness** | `InvokeHarness` via `AGENTCORE_HARNESS_ARN`; Pi-like partnership prompt; S3 file bridge |
+
+Set in `config/agent.env` or compose:
+
+```bash
+AGENT_ORCHESTRATOR=langgraph
+# AGENT_ORCHESTRATOR=agentcore
+# AGENTCORE_RUNTIME_URL=https://...
+# AGENT_ORCHESTRATOR=agentcore-harness
+# AGENTCORE_HARNESS_ARN=arn:aws:bedrock-agentcore:...
+# AGENTCORE_HARNESS_S3_INPUT_PREFIX=s3://bucket/harness-inputs/
+# LANGGRAPH_REQUIRE_REVIEW_APPROVAL=true   # gate review_apply until approve_review_apply tool
+```
+
+Headless LangGraph spike: `python agent-redact/redaction_langgraph/headless_pass1.py --pdf path/to.pdf --direct-tool`
+
+### Environment variables
+
+Copy [`config/agent.env.example`](../../../config/agent.env.example) to `config/agent.env` (gitignored) or set on the host before `docker compose up`:
+
+| Variable | Purpose |
+|----------|---------|
+| `AGENT_DEFAULT_PROVIDER` | `llama-cpp` \| `google-gemini` \| `amazon-bedrock` |
+| `AGENT_DEFAULT_MODEL` | Model id within provider |
+| `AGENT_LLAMA_BASE_URL` | Local OpenAI-compatible URL (default `http://llama-inference:8080/v1`) |
+| `AGENT_LLAMA_MODEL_ID` | Local model id |
+| `GEMINI_API_KEY` / `GOOGLE_API_KEY` | Gemini API key |
+| `AWS_REGION` / `AWS_DEFAULT_REGION` | Bedrock region |
+| `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` | Bedrock credentials (when not using SSO) |
+| `AWS_PROFILE` | Named profile for SSO / shared credentials file (**required for Pi Bedrock with SSO**) |
+| `AGENT_AWS_PROFILE` | Alternative to `AWS_PROFILE`; also used to auto-select profile when only `~/.aws` is mounted |
+| `RUN_AWS_FUNCTIONS` | When `True`, use the AWS default credential chain (SSO, profile, role) |
+| `PRIORITISE_SSO_OVER_AWS_ENV_ACCESS_KEYS` | When `True` with `RUN_AWS_FUNCTIONS`, prefer SSO/chain over static env keys (default `True`, same as main app) |
+| `AGENT_MAX_PAGES` | Maximum PDF pages allowed per redaction upload (falls back to `MAX_PAGES` / `MAX_DOC_PAGES`, default `3000`) |
+| `AGENT_MAX_RETRIES` | Gemini quota / rate-limit retries for Pi auto-retry and Gradio backoff (default `5`; alias `AGENT_QUOTA_RETRY_ATTEMPTS`) |
+| `AGENT_QUOTA_RETRY_DELAY_S` | Seconds between Gradio quota retries (default `60`) |
+| `AGENT_COMPACTION_ENABLED` | Pi session auto-compaction in `settings.json` (`true` / `false`; unset uses template default, enabled) |
+| `AGENT_COMPACTION_RESERVE_TOKENS` | Optional compaction `reserveTokens` (default `32768` from template) |
+| `AGENT_COMPACTION_KEEP_RECENT_TOKENS` | Optional compaction `keepRecentTokens` (default `20000` from template) |
+| `AGENT_CODING_AGENT_DIR` | Writable directory for generated `models.json` / `settings.json` (HF/ECS: `/tmp/agent-coding`) |
+| `AGENT_SESSION_DIR` | Pi session JSONL directory (HF/ECS: `/tmp/agent-sessions`) |
+| `PI_CODING_AGENT_DIR` | Pi CLI config directory (auto-mirrored from `AGENT_CODING_AGENT_DIR` at startup) |
+| `PI_CODING_AGENT_SESSION_DIR` | Pi CLI session directory (auto-mirrored from `AGENT_SESSION_DIR`; overrides `settings.json`) |
+
+### Usage logging (CSV / DynamoDB / S3)
+
+Each completed Pi agent run (chat message or redaction task) writes **one row** to the **same usage log schema** as the main redaction app (`USAGE_LOG_FILE_NAME`, `USAGE_LOGS_FOLDER`, `S3_USAGE_LOGS_FOLDER`, `USAGE_LOG_DYNAMODB_TABLE_NAME`). Key fields:
+
+| Log column | Pi agent value |
+|------------|----------------|
+| `task` | `agent` |
+| `llm_model_name` | Pi provider/model (e.g. `amazon-bedrock/anthropic.claude-sonnet-4-6`) |
+| `text_extraction_method` / `pii_detection_method` | From redaction task settings when applicable |
+| `actual_time_taken_number` | Wall-clock seconds for the Pi RPC turn |
+| `total_page_count` | Pages in scope for PDF redaction tasks |
+| `llm_total_input_tokens` / `llm_total_output_tokens` | Pi orchestration LLM usage for that turn (from Pi `get_session_stats` delta, or assistant `usage` in session JSONL). Includes cache read/write in the input column. **VLM/tokens from doc_redaction Pass 1 are not included** (those stay on the main app usage log when you run redaction there directly). |
+
+Toggle with `SAVE_LOGS_TO_CSV`, `SAVE_LOGS_TO_DYNAMODB`, and `RUN_AWS_FUNCTIONS` (required for S3 log upload). Access logs on session load use the main app access log paths separately.
+
+At startup, if only `GOOGLE_API_KEY` is set, it is mirrored to `GEMINI_API_KEY` for Pi.
+
+### Gradio UI
+
+Open **http://localhost:7862** → **Agent backend** accordion:
+
+- Select provider and model
+- Optionally enter Gemini / AWS credentials (**session-only** — not written to disk)
+- Click **Apply backend** — regenerates config, restarts the Pi RPC subprocess, and starts a new session
+
+Credential fields are cleared after apply.
+
+## Local model id
+
+After the llama.cpp service is healthy, confirm the model id:
+
+```bash
+curl http://localhost:8000/v1/models
+```
+
+If the returned `id` differs from `unsloth/Qwen3.6-27B-MTP-GGUF`, set `AGENT_LLAMA_MODEL_ID` in `config/agent.env` or compose environment and restart `pi-agent`.
+
+### llama.cpp / llama-swap and back-to-back redaction tasks
+
+If the **first** redaction task succeeds but a **second** task in the same browser session kills the llama server (`Killed`, `saving prompt with length 69804`, `proxy error: EOF`, `502`):
+
+1. **Oversized Pi session** — the orchestration agent kept the full first run (tool logs, bash output) in context (~70k tokens). The Gradio UI **restarts the Pi RPC process** and **clears the chat panel** on **page reload** and before each **Start redaction task** (same behaviour). Workspace files are unchanged. Use **New session** before a follow-up **chat** turn if you still hit context limits.
+2. **llama.cpp OOM** — a second task that reuses the first run’s context can try to allocate multi‑GiB KV state (`total state size = 3322 MiB` in logs) and be killed by the OS. A clean Pi process keeps the orchestration prompt small.
+3. **llama-swap GPU monitor** — on newer NVIDIA drivers, older llama-swap builds fail `nvidia-smi -loop` and can log `failed reading from gpuCh`. Upgrade to [llama-swap v213+](https://github.com/mostlygeek/llama-swap) (or disable performance monitoring in your swap config).
+4. **Concurrent load** — Pi orchestration and doc_redaction VLM may share one llama endpoint; `--parallel 1` allows only one generation. Wait until the first task shows **Agent finished** before starting another.
+
+For Gemma 4 31B, `pi-agent-gemma-31b` sets lower compaction defaults (`AGENT_COMPACTION_RESERVE_TOKENS=16384`) to match `AGENT_LLAMA_CONTEXT_WINDOW=65536`.
+
+## In-container URLs for task prompts
+
+When filling [`skills/doc-redaction-task-prompt/TASK_PROMPT_TEMPLATE.md`](../../../skills/doc-redaction-task-prompt/TASK_PROMPT_TEMPLATE.md) inside the Pi container, use:
+
+| Placeholder | In-container value |
+|-------------|-------------------|
+| `{GRADIO_URL}` | `http://redaction-app-llama:7860` |
+| `{VLM_BASE_URL}` | `http://llama-inference:8080` |
+| `{INPUT_PATH}` | `/home/user/app/workspace/{session_hash}/{FILE_NAME}` (when `AGENT_SESSION_WORKSPACE=true`) |
+| `{OUTPUT_BASE}` | `/home/user/app/workspace/{session_hash}/redact/{FILE_NAME}/` |
+
+Host-side examples (`host.docker.internal`, `localhost:7861`) do not apply inside the compose network.
+
+## Usage
+
+Start the stack (27B profile):
+
+```powershell
+docker compose -f docker-compose_llama_agentic.yml --profile 27b_36 up -d --build
+```
+
+Interactive Pi TUI:
+
+```powershell
+docker compose -f docker-compose_llama_agentic.yml exec -it pi-agent pi
+```
+
+Gradio chat UI (browser):
+
+Open **http://localhost:7862**. Use the **Redaction task** panel to upload a document, enter bullet-point requirements, and click **Start redaction task**. Pi receives the filled prompt from [`skills/Example prompt partnership.txt`](../../../skills/Example%20prompt%20partnership.txt) (file copied to `/home/user/app/workspace/`). The full prompt appears in the chat; Pi’s reply streams in the chat panel.
+
+The UI also shows:
+
+- **Agent backend** — switch between local, Gemini, and Bedrock
+- **Chat** — streamed assistant text
+- **Activity** — agent/turn lifecycle, compaction, auto-retry, tool start/end
+- **Tool output** — live bash/read output from `tool_execution_update` / `tool_execution_end`
+- **Thinking** — optional stream (`AGENT_GRADIO_SHOW_THINKING=true`)
+- **Abort** — sends Pi RPC `abort` and cancels the in-flight Gradio handler
+- **Workspace output files** — browse and download redaction artifacts
+
+Optional env vars on `pi-agent`: `AGENT_GRADIO_SHOW_THINKING`, `AGENT_GRADIO_SHOW_TOOL_OUTPUT`, `AGENT_GRADIO_TOOL_OUTPUT_MAX`, `AGENT_GRADIO_ACTIVITY_MAX_LINES`.
+
+When a Pi run completes, the chat shows an **Agent finished** (or **Agent stopped**) line, a Gradio info toast appears, and the browser tab title flashes for ~15 seconds. Desktop notifications are shown when the browser has granted notification permission (requested on first click/keypress in the Pi UI).
+
+Run the UI locally (outside Docker):
+
+```powershell
+cd agent-redact/shared
+pip install -r ../requirements_agent.txt
+# Pi orchestration subprocess (required for Apply backend / chat):
+npm install -g @earendil-works/pi-coding-agent
+python ../pi/pi_agent_config.py
+python gradio_app.py
+```
+
+**Apply backend** starts `pi --mode rpc`. If you see `FileNotFoundError` / “Pi CLI not found”, install Node.js, run the `npm install` line above, and ensure `pi` (or `pi.cmd` on Windows) is on `PATH`. Optional: `AGENT_EXECUTABLE=C:\Users\you\AppData\Roaming\npm\pi.cmd` in `config/agent.env`.
+
+RPC mode (automation, no Gradio):
+
+```powershell
+docker compose -f docker-compose_llama_agentic.yml exec -T pi-agent pi --mode rpc
+```
+
+Skills are synced from the repo `skills/` tree into **`{AGENT_WORKSPACE_DIR}/.pi/skills/`** on startup (read-only). Pi runs with `cwd` in the user’s session subfolder and `--no-skills` so it does not load skills from the git checkout. Use `/skill:doc-redaction-app` etc. Set `AGENT_SKILLS_RESYNC=true` to refresh copies from the repo.
+
+Sessions persist in the **`pi-agent-sessions`** Docker volume at **`~/.pi/agent/sessions/`** (Pi’s default session location inside the container). Override with `AGENT_SESSION_DIR` if needed.
+
+On **HF Space** (`AGENT_DEPLOYMENT_PROFILE=hf-space`), sessions go to **`/tmp/agent-sessions`** instead (ephemeral; lost on restart).
+
+## Python dependencies
+
+The agent image installs [`requirements_agent.txt`](../requirements_agent.txt) — Gradio UI + `gradio-client`, HTTP clients, CSV/PDF review helpers (`pandas`, `pymupdf`), and common utilities. It **does not** include spaCy, Presidio, or OCR; heavy redaction runs in `redaction-app-llama`.
+
+Rebuild after changing that file:
+
+```powershell
+docker compose -f docker-compose_llama_agentic.yml --profile 27b_36 build pi-agent
+```
+
+## HF Space profile (remote redaction backend)
+
+Set `AGENT_DEPLOYMENT_PROFILE=hf-space` to run the Pi Gradio UI as a **Hugging Face Docker Space** that orchestrates with **Gemini only** and calls a **remote** doc_redaction Space over HTTPS.
+
+| Area | HF Space value |
+|------|----------------|
+| Pi LLM | Gemini only (`AGENT_DEFAULT_PROVIDER=google-gemini`) |
+| Redaction app | `DOC_REDACTION_GRADIO_URL` (default `https://seanpedrickcase-document-redaction.hf.space`) |
+| Auth to redaction | `HF_TOKEN` / `DOC_REDACTION_HF_TOKEN` (Space secret + optional UI override) |
+| Text extraction / PII | Locked to `Local model - selectable text` + `Local` |
+| VLM faces / signatures | Disabled |
+| Port | `7860` |
+| Pi session logs | `/tmp/agent-sessions` (`AGENT_SESSION_DIR`; ephemeral) |
+
+Package and Dockerfile: [`agent-redact/pi-agent/`](../../pi-agent/). Pushes to [agentic_document_redaction](https://huggingface.co/spaces/seanpedrickcase/agentic_document_redaction) on **`dev`** branch via [`.github/workflows/sync-pi-agent-space.yml`](../../../.github/workflows/sync-pi-agent-space.yml) (GitHub secrets: `HF_TOKEN`, `HF_USERNAME`, `HF_EMAIL`).
+
+Local build test from monorepo root:
+
+```powershell
+docker build -f agent-redact/pi-agent/Dockerfile --target runtime -t pi-agent-hf-space .
+docker run --rm -p 7860:7860 -e GEMINI_API_KEY=... -e HF_TOKEN=... pi-agent-hf-space
+```
+
+Pi uses `gradio_client` + `agent-redact/shared/remote_redaction.py` to upload/download from the remote Space; prompts include `{REMOTE_BACKEND_GUIDANCE}` (see [`redaction_prompt.py`](../../shared/redaction_prompt.py)).

agent-redact/pi/agent/models.json

@@ -0,0 +1,31 @@
+{
+  "providers": {
+    "llama-cpp": {
+      "baseUrl": "http://llama-inference:8080/v1",
+      "api": "openai-completions",
+      "apiKey": "llama-cpp",
+      "compat": {
+        "supportsDeveloperRole": false,
+        "supportsReasoningEffort": false,
+        "supportsUsageInStreaming": false,
+        "maxTokensField": "max_tokens"
+      },
+      "models": [
+        {
+          "id": "unsloth/Qwen3.6-27B-MTP-GGUF",
+          "name": "Qwen 3.6 27B (local)",
+          "reasoning": false,
+          "input": ["text", "image"],
+          "contextWindow": 114688,
+          "maxTokens": 32768,
+          "cost": {
+            "input": 0,
+            "output": 0,
+            "cacheRead": 0,
+            "cacheWrite": 0
+          }
+        }
+      ]
+    }
+  }
+}

agent-redact/pi/agent/settings.json

@@ -0,0 +1,32 @@
+{
+  "defaultProvider": "llama-cpp",
+  "defaultModel": "unsloth/Qwen3.6-27B-MTP-GGUF",
+  "defaultThinkingLevel": "off",
+  "hideThinkingBlock": true,
+  "compaction": {
+    "enabled": true,
+    "reserveTokens": 32768,
+    "keepRecentTokens": 20000
+  },
+  "branchSummary": {
+    "skipPrompt": true,
+    "reserveTokens": 32768
+  },
+  "retry": {
+    "enabled": true,
+    "maxRetries": 5,
+    "baseDelayMs": 2000,
+    "provider": {
+      "timeoutMs": 3600000,
+      "maxRetries": 5,
+      "maxRetryDelayMs": 60000
+    }
+  },
+  "enableSkillCommands": true,
+  "sessionDir": "sessions",
+  "steeringMode": "one-at-a-time",
+  "followUpMode": "one-at-a-time",
+  "terminal": {
+    "showTerminalProgress": false
+  }
+}

agent-redact/pi/pi_agent_config.py

@@ -0,0 +1,974 @@
+"""Generate Pi agent models.json and settings.json at runtime."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+DEPLOYMENT_LOCAL = "local-docker"
+DEPLOYMENT_HF_SPACE = "hf-space"
+DEPLOYMENT_AWS_ECS = "aws-ecs"
+
+
+def resolve_agent_dir() -> Path:
+    """Directory for Pi ``models.json`` / ``settings.json`` (must be writable at runtime)."""
+    explicit = (os.environ.get("AGENT_CODING_AGENT_DIR") or "").strip()
+    if explicit:
+        return Path(explicit)
+    profile = (
+        os.environ.get("AGENT_DEPLOYMENT_PROFILE", DEPLOYMENT_LOCAL).strip().lower()
+    )
+    # HF Space and ECS often use a read-only root FS; only mounted paths (or /tmp) are writable.
+    if profile in (DEPLOYMENT_HF_SPACE, DEPLOYMENT_AWS_ECS):
+        return Path("/tmp/agent-coding")
+    return Path.home() / ".pi" / "agent"
+
+
+# Back-compat alias; prefer resolve_agent_dir() when env may change after import.
+AGENT_DIR = resolve_agent_dir()
+TEMPLATE_DIR = Path(__file__).resolve().parent / "agent"
+SETTINGS_TEMPLATE = TEMPLATE_DIR / "settings.json"
+
+DEPLOYMENT_PROFILE = (
+    os.environ.get("AGENT_DEPLOYMENT_PROFILE", DEPLOYMENT_LOCAL).strip().lower()
+)
+
+
+def pi_max_retries() -> int:
+    """Max retries for Pi auto-retry and Gradio quota backoff (env: AGENT_MAX_RETRIES, default 5)."""
+    raw = (
+        os.environ.get("AGENT_QUOTA_RETRY_ATTEMPTS")
+        or os.environ.get("AGENT_MAX_RETRIES")
+        or "5"
+    ).strip()
+    return int(raw)
+
+
+def _apply_retry_settings(
+    settings: dict[str, Any],
+    *,
+    provider: str,
+) -> None:
+    """Write Pi ``settings.json`` retry block (cloud providers use longer delays)."""
+    max_retries = pi_max_retries()
+    use_long_delays = (
+        provider == PROVIDER_GEMINI
+        or provider == PROVIDER_BEDROCK
+        or is_hf_space_profile()
+        or is_aws_ecs_profile()
+    )
+    base_delay_ms = 2000
+    max_delay_ms = 60000
+    if use_long_delays:
+        default_base_ms = int(os.environ.get("AGENT_QUOTA_RETRY_DELAY_S", "60")) * 1000
+        default_max_ms = int(default_base_ms * 1.5)
+        if provider == PROVIDER_BEDROCK or (
+            is_aws_ecs_profile() and not is_hf_space_profile()
+        ):
+            prefix = "AGENT_BEDROCK"
+        else:
+            prefix = "AGENT_GEMINI"
+        base_delay_ms = int(
+            os.environ.get(f"{prefix}_RETRY_BASE_DELAY_MS")
+            or os.environ.get("AGENT_GEMINI_RETRY_BASE_DELAY_MS", str(default_base_ms))
+        )
+        max_delay_ms = int(
+            os.environ.get(f"{prefix}_RETRY_MAX_DELAY_MS")
+            or os.environ.get("AGENT_GEMINI_RETRY_MAX_DELAY_MS", str(default_max_ms))
+        )
+    settings["retry"] = {
+        "enabled": True,
+        "maxRetries": max_retries,
+        "baseDelayMs": base_delay_ms,
+        "provider": {
+            "timeoutMs": 3600000,
+            "maxRetries": max_retries,
+            "maxRetryDelayMs": max_delay_ms,
+        },
+    }
+
+
+PROVIDER_LLAMA = "llama-cpp"
+PROVIDER_GEMINI = "google-gemini"
+PROVIDER_BEDROCK = "amazon-bedrock"
+
+PROVIDER_LABELS: dict[str, str] = {
+    PROVIDER_LLAMA: "Local (llama-cpp)",
+    PROVIDER_GEMINI: "Gemini",
+    PROVIDER_BEDROCK: "AWS Bedrock",
+}
+
+# Pi RPC ``get_state`` provider ids (e.g. ``google``) → canonical config ids.
+_PI_RPC_PROVIDER_ALIASES: dict[str, str] = {
+    "google": PROVIDER_GEMINI,
+    "gemini": PROVIDER_GEMINI,
+    "bedrock": PROVIDER_BEDROCK,
+    "llama": PROVIDER_LLAMA,
+}
+
+
+def is_hf_space_profile() -> bool:
+    profile = (
+        os.environ.get("AGENT_DEPLOYMENT_PROFILE", DEPLOYMENT_LOCAL).strip().lower()
+    )
+    return profile == DEPLOYMENT_HF_SPACE
+
+
+def is_aws_ecs_profile() -> bool:
+    profile = (
+        os.environ.get("AGENT_DEPLOYMENT_PROFILE", DEPLOYMENT_LOCAL).strip().lower()
+    )
+    return profile == DEPLOYMENT_AWS_ECS
+
+
+def uses_split_redaction_backend() -> bool:
+    """
+    True when Pi and doc_redaction run in separate containers (no shared output disk).
+
+    HF Space and AWS ECS use Gradio HTTP download; local-docker typically shares a host
+    volume. Override with ``AGENT_REDACTION_SPLIT_BACKEND=true|false``.
+    """
+    explicit = (os.environ.get("AGENT_REDACTION_SPLIT_BACKEND") or "").strip().lower()
+    if explicit in {"1", "true", "yes", "on"}:
+        return True
+    if explicit in {"0", "false", "no", "off"}:
+        return False
+    return is_hf_space_profile() or is_aws_ecs_profile()
+
+
+def resolve_llama_base_url() -> str:
+    """
+    OpenAI-compatible base URL for Pi's ``llama-cpp`` provider (includes ``/v1``).
+
+    Reads ``AGENT_LLAMA_BASE_URL``; also accepts legacy aliases
+    ``AGENT_LLAMA_MODE_BASE_URL`` and ``AGENT_LLAMA_MODE__BASE_URL``.
+    """
+    for key in (
+        "AGENT_LLAMA_BASE_URL",
+        "AGENT_LLAMA_MODE_BASE_URL",
+    ):
+        raw = (os.environ.get(key) or "").strip().rstrip("/")
+        if raw:
+            return raw if raw.endswith("/v1") else f"{raw}/v1"
+    return "http://llama-inference:8080/v1"
+
+
+LLAMA_BASE_URL = resolve_llama_base_url()
+LLAMA_MODEL_ID = os.environ.get("AGENT_LLAMA_MODEL_ID", "unsloth/Qwen3.6-27B-MTP-GGUF")
+LLAMA_CONTEXT = int(os.environ.get("AGENT_LLAMA_CONTEXT_WINDOW", "114688"))
+LLAMA_MAX_TOKENS = int(os.environ.get("AGENT_LLAMA_MAX_TOKENS", "32768"))
+
+GEMINI_MODELS: tuple[tuple[str, str, int, bool], ...] = (
+    ("gemini-flash-lite-latest", "Gemini Flash Lite", 1048576, False),
+    ("gemini-flash-latest", "Gemini Flash", 1048576, True),
+    ("gemini-pro-latest", "Gemini Pro", 1048576, True),
+)
+
+BEDROCK_MODELS: tuple[tuple[str, str, int, bool], ...] = (
+    (
+        "anthropic.claude-sonnet-4-6",
+        "Anthropic Claude Sonnet 4.6 (Bedrock)",
+        1000000,
+        True,
+    ),
+    ("amazon.nova-pro-v1:0", "Amazon Nova Pro (Bedrock)", 300000, False),
+    (
+        "nvidia.nemotron-super-3-120b",
+        "NVIDIA Nemotron Super 3 120B (Bedrock)",
+        262000,
+        False,
+    ),
+    ("mistral.devstral-2-123b", "Mistral Devstral 2 123B (Bedrock)", 256000, False),
+)
+
+PROVIDER_MODELS: dict[str, list[str]] = {
+    PROVIDER_LLAMA: [LLAMA_MODEL_ID],
+    PROVIDER_GEMINI: [model_id for model_id, _, _, _ in GEMINI_MODELS],
+    PROVIDER_BEDROCK: [model_id for model_id, _, _, _ in BEDROCK_MODELS],
+}
+
+DEFAULT_MODEL_BY_PROVIDER: dict[str, str] = {
+    PROVIDER_LLAMA: LLAMA_MODEL_ID,
+    PROVIDER_GEMINI: GEMINI_MODELS[0][0],  # Gemini Flash Lite
+    PROVIDER_BEDROCK: "anthropic.claude-sonnet-4-6",
+}
+
+
+def get_default_provider() -> str:
+    """Current default Pi provider (reads ``AGENT_DEFAULT_PROVIDER`` from env each call)."""
+    if is_hf_space_profile():
+        return PROVIDER_GEMINI
+    raw = (os.environ.get("AGENT_DEFAULT_PROVIDER") or "").strip()
+    if raw in PROVIDER_MODELS:
+        return raw
+    if is_aws_ecs_profile():
+        return PROVIDER_BEDROCK
+    return PROVIDER_LLAMA
+
+
+DEFAULT_PROVIDER = get_default_provider()
+
+
+def _catalog_contains_model(model_id: str, provider: str) -> bool:
+    """True when *model_id* is listed for a non-llama *provider*."""
+    return model_id in PROVIDER_MODELS.get(provider, ())
+
+
+_env_default_model = (os.environ.get("AGENT_DEFAULT_MODEL") or "").strip()
+if _env_default_model and (
+    DEFAULT_PROVIDER == PROVIDER_LLAMA
+    or _catalog_contains_model(_env_default_model, DEFAULT_PROVIDER)
+):
+    DEFAULT_MODEL = _env_default_model
+else:
+    DEFAULT_MODEL = DEFAULT_MODEL_BY_PROVIDER.get(DEFAULT_PROVIDER, LLAMA_MODEL_ID)
+
+
+def llama_model_id() -> str:
+    """Active llama-cpp model id (runtime ``AGENT_LLAMA_MODEL_ID`` or startup default)."""
+    return (
+        os.environ.get("AGENT_LLAMA_MODEL_ID") or LLAMA_MODEL_ID
+    ).strip() or LLAMA_MODEL_ID
+
+
+def resolved_default_model(provider: str, *, override: str | None = None) -> str:
+    """
+    Pick the default model id for a provider.
+
+    Order: explicit override → ``AGENT_DEFAULT_MODEL`` when valid for *provider* →
+    built-in per-provider default (llama uses ``AGENT_LLAMA_MODEL_ID``).
+    """
+    if override and override.strip():
+        return override.strip()
+    normalized = normalize_provider(provider)
+    env_model = (os.environ.get("AGENT_DEFAULT_MODEL") or "").strip()
+    active_provider = normalize_provider(get_default_provider())
+    if env_model:
+        if normalized == PROVIDER_LLAMA:
+            if active_provider == PROVIDER_LLAMA:
+                return env_model
+        elif _catalog_contains_model(env_model, normalized):
+            return env_model
+    if normalized == PROVIDER_LLAMA:
+        return llama_model_id()
+    return DEFAULT_MODEL_BY_PROVIDER.get(normalized, LLAMA_MODEL_ID)
+
+
+def normalize_backend_model(provider: str, model_id: str | None) -> str:
+    """
+    Resolve a UI/backend model selection to a concrete model id.
+
+    llama-cpp accepts any non-empty id (llama-swap / custom OpenAI model names).
+    Other providers must match the static catalog.
+    """
+    normalized = normalize_provider(provider)
+    model = (model_id or default_model_for_provider(normalized)).strip()
+    if not model:
+        return default_model_for_provider(normalized)
+    if normalized == PROVIDER_LLAMA:
+        return model
+    if model in models_for_provider(normalized):
+        return model
+    return default_model_for_provider(normalized)
+
+
+def _zero_cost() -> dict[str, int]:
+    return {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0}
+
+
+def _model_entry(
+    model_id: str,
+    name: str,
+    *,
+    context_window: int,
+    max_tokens: int,
+    reasoning: bool,
+    image_input: bool = True,
+) -> dict[str, Any]:
+    inputs = ["text", "image"] if image_input else ["text"]
+    return {
+        "id": model_id,
+        "name": name,
+        "reasoning": reasoning,
+        "input": inputs,
+        "contextWindow": context_window,
+        "maxTokens": max_tokens,
+        "cost": _zero_cost(),
+    }
+
+
+def _llama_provider() -> dict[str, Any]:
+    model_id = llama_model_id()
+    return {
+        "baseUrl": LLAMA_BASE_URL,
+        "api": "openai-completions",
+        "apiKey": "llama-cpp",
+        "compat": {
+            "supportsDeveloperRole": False,
+            "supportsReasoningEffort": False,
+            "supportsUsageInStreaming": False,
+            "maxTokensField": "max_tokens",
+        },
+        "models": [
+            _model_entry(
+                model_id,
+                f"Local ({model_id})",
+                context_window=LLAMA_CONTEXT,
+                max_tokens=LLAMA_MAX_TOKENS,
+                reasoning=False,
+            )
+        ],
+    }
+
+
+def _gemini_provider() -> dict[str, Any]:
+    return {
+        "baseUrl": "https://generativelanguage.googleapis.com/v1beta",
+        "api": "google-generative-ai",
+        "apiKey": "GEMINI_API_KEY",
+        "models": [
+            _model_entry(
+                model_id, name, context_window=ctx, max_tokens=8192, reasoning=reasoning
+            )
+            for model_id, name, ctx, reasoning in GEMINI_MODELS
+        ],
+    }
+
+
+def _bedrock_region() -> str:
+    return (
+        os.environ.get("AWS_REGION")
+        or os.environ.get("AWS_DEFAULT_REGION")
+        or "eu-west-2"
+    )
+
+
+_AWS_CREDENTIAL_ENV_KEYS: tuple[str, ...] = (
+    "AWS_ACCESS_KEY_ID",
+    "AWS_SECRET_ACCESS_KEY",
+    "AWS_SESSION_TOKEN",
+    "AWS_ACCESS_KEY",
+    "AWS_SECRET_KEY",
+)
+_AWS_PROFILE_ENV_KEYS: tuple[str, ...] = ("AWS_PROFILE", "AGENT_AWS_PROFILE")
+
+
+def _env_flag(name: str, *, default: bool = False) -> bool:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _strip_empty_env_vars(names: tuple[str, ...]) -> None:
+    for name in names:
+        if not (os.environ.get(name) or "").strip():
+            os.environ.pop(name, None)
+
+
+def _mirror_legacy_aws_key_env_vars() -> None:
+    if not (os.environ.get("AWS_ACCESS_KEY_ID") or "").strip():
+        legacy = (os.environ.get("AWS_ACCESS_KEY") or "").strip()
+        if legacy:
+            os.environ["AWS_ACCESS_KEY_ID"] = legacy
+    if not (os.environ.get("AWS_SECRET_ACCESS_KEY") or "").strip():
+        legacy = (os.environ.get("AWS_SECRET_KEY") or "").strip()
+        if legacy:
+            os.environ["AWS_SECRET_ACCESS_KEY"] = legacy
+
+
+def _has_explicit_aws_access_keys() -> bool:
+    access = (
+        os.environ.get("AWS_ACCESS_KEY_ID") or os.environ.get("AWS_ACCESS_KEY") or ""
+    ).strip()
+    secret = (
+        os.environ.get("AWS_SECRET_ACCESS_KEY")
+        or os.environ.get("AWS_SECRET_KEY")
+        or ""
+    ).strip()
+    return bool(access and secret)
+
+
+def _aws_config_path() -> Path | None:
+    explicit = (os.environ.get("AWS_CONFIG_FILE") or "").strip()
+    if explicit:
+        path = Path(explicit).expanduser()
+        return path if path.is_file() else None
+    home = Path(os.environ.get("HOME", "/home/user"))
+    path = home / ".aws" / "config"
+    return path if path.is_file() else None
+
+
+def _discover_aws_profile_from_config() -> str | None:
+    """Return an AWS profile name for Pi/Bedrock when only ~/.aws is mounted."""
+    explicit = (os.environ.get("AGENT_AWS_PROFILE") or "").strip()
+    if not explicit:
+        explicit = (os.environ.get("AWS_PROFILE") or "").strip()
+    if explicit:
+        return explicit
+
+    path = _aws_config_path()
+    if not path:
+        return None
+
+    current_profile: str | None = None
+    sso_profiles: list[str] = []
+    all_profiles: list[str] = []
+
+    for raw_line in path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#") or line.startswith(";"):
+            continue
+        if line == "[default]":
+            current_profile = "default"
+            all_profiles.append("default")
+            continue
+        if line.startswith("[profile ") and line.endswith("]"):
+            current_profile = line[len("[profile ") : -1].strip()
+            if current_profile:
+                all_profiles.append(current_profile)
+            continue
+        if current_profile and line.startswith("sso_session"):
+            sso_profiles.append(current_profile)
+
+    if sso_profiles:
+        return sso_profiles[0]
+    if "default" in all_profiles:
+        return "default"
+    return all_profiles[0] if all_profiles else None
+
+
+def _region_from_aws_config(profile: str | None = None) -> str | None:
+    """Read ``region =`` from a profile block in ``~/.aws/config``."""
+    path = _aws_config_path()
+    if not path:
+        return None
+
+    target = (profile or _discover_aws_profile_from_config() or "").strip()
+    if not target:
+        return None
+
+    current_profile: str | None = None
+    for raw_line in path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#") or line.startswith(";"):
+            continue
+        if line == "[default]":
+            current_profile = "default"
+            continue
+        if line.startswith("[profile ") and line.endswith("]"):
+            current_profile = line[len("[profile ") : -1].strip()
+            continue
+        if current_profile != target:
+            continue
+        if line.startswith("region"):
+            _, _, value = line.partition("=")
+            region = value.strip()
+            if region:
+                return region
+    return None
+
+
+def _ensure_aws_region_env() -> None:
+    """Ensure AWS SDK env has a non-empty region (profile config, then eu-west-2)."""
+    _strip_empty_env_vars(("AWS_REGION", "AWS_DEFAULT_REGION"))
+    region = (
+        os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or ""
+    ).strip()
+    if not region:
+        profile = (os.environ.get("AWS_PROFILE") or "").strip()
+        region = (_region_from_aws_config(profile) or "").strip()
+    if not region:
+        region = _bedrock_region()
+    os.environ["AWS_REGION"] = region
+    os.environ["AWS_DEFAULT_REGION"] = region
+
+
+def _pi_bedrock_auth_visible() -> bool:
+    """True when Pi's amazon-bedrock provider would detect configured auth."""
+    if (os.environ.get("AWS_PROFILE") or "").strip():
+        return True
+    if _has_explicit_aws_access_keys():
+        return True
+    if (os.environ.get("AWS_BEARER_TOKEN_BEDROCK") or "").strip():
+        return True
+    return False
+
+
+def _ensure_pi_bedrock_auth_env() -> None:
+    """
+    Pi checks env vars (not ~/.aws alone) before Bedrock is usable.
+
+    When SSO credentials live in a mounted ``~/.aws`` tree, set ``AWS_PROFILE``
+    so Pi passes its auth preflight and the AWS SDK loads the profile.
+    """
+    if _pi_bedrock_auth_visible():
+        return
+    profile = _discover_aws_profile_from_config()
+    if profile:
+        os.environ["AWS_PROFILE"] = profile
+
+
+def configure_aws_credentials(
+    *,
+    session_access_key_id: str | None = None,
+    session_secret_access_key: str | None = None,
+    session_session_token: str | None = None,
+) -> None:
+    """
+    Align Pi Bedrock AWS env with doc_redaction SSO/key priority.
+
+    Mirrors ``tools/file_redaction.py``: when ``RUN_AWS_FUNCTIONS`` is enabled,
+    prefer the default credential chain (SSO profile, instance role, etc.) over
+    static env keys when ``PRIORITISE_SSO_OVER_AWS_ENV_ACCESS_KEYS`` is true.
+    Explicit UI session keys from **Apply backend** always win.
+    """
+    _strip_empty_env_vars(_AWS_CREDENTIAL_ENV_KEYS)
+    _strip_empty_env_vars(_AWS_PROFILE_ENV_KEYS)
+    _mirror_legacy_aws_key_env_vars()
+
+    session_explicit = bool(
+        session_access_key_id
+        and session_access_key_id.strip()
+        and session_secret_access_key
+        and session_secret_access_key.strip()
+    )
+    if session_explicit:
+        os.environ["AWS_ACCESS_KEY_ID"] = session_access_key_id.strip()
+        os.environ["AWS_SECRET_ACCESS_KEY"] = session_secret_access_key.strip()
+        if session_session_token and session_session_token.strip():
+            os.environ["AWS_SESSION_TOKEN"] = session_session_token.strip()
+        else:
+            os.environ.pop("AWS_SESSION_TOKEN", None)
+        _ensure_aws_region_env()
+        return
+
+    run_aws = _env_flag("RUN_AWS_FUNCTIONS")
+    prioritise_sso = _env_flag("PRIORITISE_SSO_OVER_AWS_ENV_ACCESS_KEYS", default=True)
+
+    if run_aws and prioritise_sso:
+        for key in _AWS_CREDENTIAL_ENV_KEYS:
+            os.environ.pop(key, None)
+        _ensure_pi_bedrock_auth_env()
+    elif run_aws:
+        for key in _AWS_CREDENTIAL_ENV_KEYS:
+            os.environ.pop(key, None)
+        _ensure_pi_bedrock_auth_env()
+
+    # Propagate AGENT_AWS_PROFILE when only that alias is set (e.g. agent.env).
+    pi_profile = (os.environ.get("AGENT_AWS_PROFILE") or "").strip()
+    if pi_profile and not (os.environ.get("AWS_PROFILE") or "").strip():
+        os.environ["AWS_PROFILE"] = pi_profile
+
+    _ensure_aws_region_env()
+
+
+def _aws_credential_status() -> str:
+    if _has_explicit_aws_access_keys():
+        return "access keys"
+    profile = (os.environ.get("AWS_PROFILE") or "").strip()
+    if profile:
+        return f"profile {profile}"
+    if (os.environ.get("AWS_BEARER_TOKEN_BEDROCK") or "").strip():
+        return "Bedrock bearer token"
+    if _aws_config_path():
+        return "SSO config mounted (profile not set)"
+    if _env_flag("RUN_AWS_FUNCTIONS"):
+        return "SSO/default chain (missing profile)"
+    return "missing"
+
+
+def _bedrock_provider() -> dict[str, Any]:
+    region = _bedrock_region()
+    return {
+        "baseUrl": f"https://bedrock-runtime.{region}.amazonaws.com",
+        "api": "bedrock-converse-stream",
+        "models": [
+            _model_entry(
+                model_id,
+                name,
+                context_window=ctx,
+                max_tokens=8192,
+                reasoning=reasoning,
+            )
+            for model_id, name, ctx, reasoning in BEDROCK_MODELS
+        ],
+    }
+
+
+def build_models_config() -> dict[str, Any]:
+    if is_hf_space_profile():
+        return {"providers": {PROVIDER_GEMINI: _gemini_provider()}}
+    return {
+        "providers": {
+            PROVIDER_LLAMA: _llama_provider(),
+            PROVIDER_GEMINI: _gemini_provider(),
+            PROVIDER_BEDROCK: _bedrock_provider(),
+        }
+    }
+
+
+def _load_settings_template() -> dict[str, Any]:
+    if SETTINGS_TEMPLATE.is_file():
+        return json.loads(SETTINGS_TEMPLATE.read_text(encoding="utf-8"))
+    return {
+        "defaultThinkingLevel": "off",
+        "hideThinkingBlock": True,
+        "compaction": {
+            "enabled": True,
+            "reserveTokens": 32768,
+            "keepRecentTokens": 20000,
+        },
+        "enableSkillCommands": True,
+        "sessionDir": "sessions",
+    }
+
+
+def _apply_compaction_settings(settings: dict[str, Any]) -> None:
+    """
+    Merge Pi session auto-compaction from env into ``settings.json``.
+
+    ``AGENT_COMPACTION_ENABLED`` — when set, overrides the template ``compaction.enabled``
+    flag (``true`` / ``false``). When unset, the template default applies (enabled).
+
+    Optional tuning: ``AGENT_COMPACTION_RESERVE_TOKENS``, ``AGENT_COMPACTION_KEEP_RECENT_TOKENS``.
+    """
+    compaction = dict(
+        settings.get("compaction")
+        or {
+            "enabled": True,
+            "reserveTokens": 32768,
+            "keepRecentTokens": 20000,
+        }
+    )
+    if os.environ.get("AGENT_COMPACTION_ENABLED") is not None:
+        compaction["enabled"] = _env_flag("AGENT_COMPACTION_ENABLED")
+    reserve = (os.environ.get("AGENT_COMPACTION_RESERVE_TOKENS") or "").strip()
+    if reserve:
+        compaction["reserveTokens"] = int(reserve)
+    elif LLAMA_CONTEXT < 100_000:
+        # Smaller local models (e.g. Gemma 4 31B at 65536): default reserve was 32768.
+        compaction["reserveTokens"] = min(16_384, max(8_192, LLAMA_CONTEXT // 4))
+    keep = (os.environ.get("AGENT_COMPACTION_KEEP_RECENT_TOKENS") or "").strip()
+    if keep:
+        compaction["keepRecentTokens"] = int(keep)
+    elif LLAMA_CONTEXT < 100_000:
+        compaction["keepRecentTokens"] = min(12_288, max(4_096, LLAMA_CONTEXT // 5))
+    settings["compaction"] = compaction
+
+
+def resolve_session_dir() -> str:
+    """Pi session JSONL directory (absolute path or relative to ``AGENT_DIR``)."""
+    explicit = os.environ.get("AGENT_SESSION_DIR", "").strip()
+    if explicit:
+        return explicit
+    if is_hf_space_profile():
+        return "/tmp/agent-sessions"
+    return "sessions"
+
+
+def ensure_session_dir(session_dir: str | None = None) -> Path:
+    """Create the Pi session directory and return its resolved absolute path."""
+    raw = (session_dir or resolve_session_dir()).strip()
+    path = Path(raw)
+    if not path.is_absolute():
+        path = (resolve_agent_dir() / path).resolve()
+    else:
+        path = path.resolve()
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def configure_pi_coding_agent_env() -> None:
+    """
+    Mirror ``AGENT_*`` paths into Pi CLI env vars.
+
+    The external ``pi`` binary reads ``PI_CODING_AGENT_DIR`` (models/settings) and
+    ``PI_CODING_AGENT_SESSION_DIR`` (session JSONL), not our ``AGENT_*`` names.
+    HF Space / ECS images write config under ``/tmp``; without this mirror Pi falls
+    back to ``~/.pi/agent``, which is often not writable on Spaces.
+    """
+    agent_dir = str(resolve_agent_dir())
+    session_dir = str(ensure_session_dir(resolve_session_dir()))
+    os.environ.setdefault("PI_CODING_AGENT_DIR", agent_dir)
+    os.environ.setdefault("PI_CODING_AGENT_SESSION_DIR", session_dir)
+
+
+def build_settings_config(
+    *,
+    default_provider: str | None = None,
+    default_model: str | None = None,
+) -> dict[str, Any]:
+    provider = default_provider or get_default_provider()
+    if provider not in PROVIDER_MODELS:
+        provider = PROVIDER_GEMINI if is_hf_space_profile() else PROVIDER_LLAMA
+    model = resolved_default_model(provider, override=default_model)
+
+    settings = _load_settings_template()
+    settings["defaultProvider"] = provider
+    settings["defaultModel"] = model
+    _apply_compaction_settings(settings)
+    session_path = ensure_session_dir(resolve_session_dir())
+    settings["sessionDir"] = session_path.as_posix()
+    if (
+        is_hf_space_profile()
+        or is_aws_ecs_profile()
+        or provider in (PROVIDER_GEMINI, PROVIDER_BEDROCK)
+    ):
+        _apply_retry_settings(settings, provider=provider)
+    from pi_workspace_skills import ensure_workspace_skills, workspace_skills_dir
+
+    ensure_workspace_skills()
+    settings["skills"] = [workspace_skills_dir().as_posix()]
+    return settings
+
+
+def write_runtime_config(
+    *,
+    agent_dir: Path | None = None,
+    default_provider: str | None = None,
+    default_model: str | None = None,
+) -> tuple[Path, Path]:
+    """Write models.json and settings.json; return their paths."""
+    provider = normalize_provider(default_provider or get_default_provider())
+    if default_provider:
+        os.environ["AGENT_DEFAULT_PROVIDER"] = provider
+    if default_model and default_model.strip():
+        model = default_model.strip()
+        os.environ["AGENT_DEFAULT_MODEL"] = model
+        if provider == PROVIDER_LLAMA:
+            os.environ["AGENT_LLAMA_MODEL_ID"] = model
+
+    target = Path(agent_dir or resolve_agent_dir())
+    target.mkdir(parents=True, exist_ok=True)
+
+    models_path = target / "models.json"
+    settings_path = target / "settings.json"
+
+    models_path.write_text(
+        json.dumps(build_models_config(), indent=2) + "\n",
+        encoding="utf-8",
+    )
+    settings_path.write_text(
+        json.dumps(
+            build_settings_config(
+                default_provider=default_provider,
+                default_model=default_model,
+            ),
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    configure_pi_coding_agent_env()
+    return models_path, settings_path
+
+
+def models_for_provider(provider: str) -> list[str]:
+    if is_hf_space_profile():
+        return list(PROVIDER_MODELS[PROVIDER_GEMINI])
+    if provider == PROVIDER_LLAMA:
+        return [llama_model_id()]
+    return list(PROVIDER_MODELS.get(provider, PROVIDER_MODELS[PROVIDER_LLAMA]))
+
+
+def default_model_for_provider(provider: str) -> str:
+    return resolved_default_model(provider)
+
+
+def normalize_provider(provider: str) -> str:
+    label_map = {label.lower(): key for key, label in PROVIDER_LABELS.items()}
+    lowered = (provider or "").strip().lower()
+    if lowered in PROVIDER_MODELS:
+        return lowered
+    if lowered in _PI_RPC_PROVIDER_ALIASES:
+        return _PI_RPC_PROVIDER_ALIASES[lowered]
+    if lowered in label_map:
+        return label_map[lowered]
+    return PROVIDER_GEMINI if is_hf_space_profile() else PROVIDER_LLAMA
+
+
+def active_model_from_pi_state(state: dict[str, Any]) -> tuple[str, str]:
+    """Return ``(normalized_provider, model_id)`` from a Pi RPC ``get_state`` payload."""
+    if not isinstance(state, dict):
+        return "", ""
+    model = state.get("model") or {}
+    if not isinstance(model, dict):
+        model = {}
+    provider_raw = str(model.get("provider") or state.get("provider") or "").strip()
+    model_id = str(model.get("id") or model.get("name") or "").strip()
+    return normalize_provider(provider_raw), model_id
+
+
+def pi_model_fallback_notice(
+    *,
+    intended_provider: str,
+    intended_model: str,
+    active_provider: str,
+    active_model: str,
+) -> str | None:
+    """
+    User-facing warning when Pi runs a different provider/model than configured.
+
+    Returns ``None`` when the active model matches the intended configuration.
+    """
+    intended_p = normalize_provider(intended_provider)
+    intended_m = (intended_model or "").strip()
+    active_p = normalize_provider(active_provider)
+    active_m = (active_model or "").strip()
+    if not active_m:
+        return None
+    if intended_p == active_p and intended_m.casefold() == active_m.casefold():
+        return None
+    intended_label = (
+        f"{PROVIDER_LABELS.get(intended_p, intended_p)} / `{intended_m or '—'}`"
+    )
+    active_label = f"{PROVIDER_LABELS.get(active_p, active_p)} / `{active_m}`"
+    llama_hint = ""
+    if intended_p == PROVIDER_LLAMA:
+        llama_hint = (
+            f" Check your llama.cpp model id (`GET {LLAMA_BASE_URL.rstrip('/')}/models`) "
+            f"and align `AGENT_DEFAULT_MODEL` / `AGENT_LLAMA_MODEL_ID` in "
+            f"`config/agent.env`."
+        )
+    cloud_hint = ""
+    if active_p == PROVIDER_GEMINI and intended_p == PROVIDER_LLAMA:
+        cloud_hint = (
+            " A Gemini API key in the container environment can trigger this fallback."
+        )
+    return (
+        f"The configured orchestration model {intended_label} was not found or could "
+        f"not be selected. Pi is using **{active_label}** instead.{llama_hint}"
+        f"{cloud_hint} Click **Apply backend** with the correct model, or recreate "
+        f"`pi-agent` after updating `config/agent.env`."
+    )
+
+
+def pi_model_fallback_notice_from_state(
+    state: dict[str, Any],
+    intended_provider: str,
+    intended_model: str,
+) -> str | None:
+    """Build a fallback notice from Pi RPC state and the configured provider/model."""
+    active_p, active_m = active_model_from_pi_state(state)
+    return pi_model_fallback_notice(
+        intended_provider=intended_provider,
+        intended_model=intended_model,
+        active_provider=active_p,
+        active_model=active_m,
+    )
+
+
+def apply_session_credentials(
+    *,
+    gemini_api_key: str | None = None,
+    hf_token: str | None = None,
+    aws_region: str | None = None,
+    aws_access_key_id: str | None = None,
+    aws_secret_access_key: str | None = None,
+    aws_session_token: str | None = None,
+) -> None:
+    """Apply session-only credential overrides to os.environ."""
+    if gemini_api_key and gemini_api_key.strip():
+        os.environ["GEMINI_API_KEY"] = gemini_api_key.strip()
+    if hf_token and hf_token.strip():
+        token = hf_token.strip()
+        os.environ["HF_TOKEN"] = token
+        os.environ["DOC_REDACTION_HF_TOKEN"] = token
+    if aws_region and aws_region.strip():
+        os.environ["AWS_REGION"] = aws_region.strip()
+        os.environ["AWS_DEFAULT_REGION"] = aws_region.strip()
+    configure_aws_credentials(
+        session_access_key_id=aws_access_key_id,
+        session_secret_access_key=aws_secret_access_key,
+        session_session_token=aws_session_token,
+    )
+
+
+def mirror_hf_token_from_env() -> None:
+    """Mirror DOC_REDACTION_HF_TOKEN or Space secret HF_TOKEN for Pi subprocess."""
+    if os.environ.get("HF_TOKEN"):
+        return
+    doc_token = os.environ.get("DOC_REDACTION_HF_TOKEN", "").strip()
+    if doc_token:
+        os.environ["HF_TOKEN"] = doc_token
+
+
+def _hf_token_status() -> str:
+    if os.environ.get("HF_TOKEN"):
+        source = (
+            "UI session" if os.environ.get("_HF_TOKEN_FROM_UI") else "env/Space secret"
+        )
+        return f"set ({source})"
+    return "missing"
+
+
+def credential_status_markdown(*, provider: str | None = None) -> str:
+    """
+    Credential summary for the active Pi provider.
+
+    ``llama-cpp`` uses the local OpenAI-compatible endpoint only (no Gemini/AWS keys).
+    Gemini and Bedrock lines appear only when that provider is selected.
+    """
+    active = normalize_provider(provider or get_default_provider())
+    orchestrator = (os.environ.get("AGENT_ORCHESTRATOR") or "pi").strip().lower()
+    if orchestrator in {"agentcore", "agentcore-harness"}:
+        try:
+            from redaction_prompt import doc_redaction_gradio_url
+
+            backend = doc_redaction_gradio_url()
+        except ImportError:
+            backend = (os.environ.get("DOC_REDACTION_GRADIO_URL") or "—").strip()
+        region = _bedrock_region()
+        return (
+            f"**Credentials:** AWS `{_aws_credential_status()}` · region `{region}` "
+            f"(AgentCore orchestration)  \n"
+            f"**Redaction tools call:** `{backend}` (sent to runtime each invoke)"
+        )
+    if is_hf_space_profile():
+        gemini = (
+            "set"
+            if os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
+            else "missing"
+        )
+        return (
+            f"**Credentials:** Gemini `{gemini}` · "
+            f"HF token (redaction backend) `{_hf_token_status()}`"
+        )
+    if active == PROVIDER_LLAMA:
+        return (
+            f"**Credentials:** local llama-cpp at `{LLAMA_BASE_URL}` "
+            f"(no API key; AWS/Gemini not used for Pi orchestration)"
+        )
+    if active == PROVIDER_GEMINI:
+        gemini = (
+            "set"
+            if os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
+            else "missing"
+        )
+        return f"**Credentials:** Gemini `{gemini}`"
+    region = _bedrock_region()
+    return f"**Credentials:** AWS `{_aws_credential_status()}` · region `{region}`"
+
+
+def provider_choices() -> list[str]:
+    if is_hf_space_profile():
+        return [PROVIDER_GEMINI]
+    return list(PROVIDER_LABELS.keys())
+
+
+def gemini_api_key_configured() -> bool:
+    return bool(os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY"))
+
+
+def provider_label(provider: str) -> str:
+    return PROVIDER_LABELS.get(provider, provider)
+
+
+if __name__ == "__main__":
+    configure_aws_credentials()
+    models_path, settings_path = write_runtime_config()
+    print(f"Wrote {models_path}")
+    print(f"Wrote {settings_path}")

agent-redact/pi/pi_rpc_client.py

@@ -0,0 +1,1016 @@
+"""Python client for Pi RPC mode (JSONL over stdin/stdout)."""
+
+from __future__ import annotations
+
+import json
+import os
+import queue
+import shutil
+import subprocess
+import sys
+import threading
+import uuid
+from collections import deque
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from typing import Any
+
+
+class PiRpcError(RuntimeError):
+    pass
+
+
+# Sentinel pushed to every pending response slot and the events queue when the
+# Pi RPC subprocess exits, so blocked waiters unblock with a clear error instead
+# of hanging forever.
+_PI_PROCESS_EXIT = object()
+
+
+# Pi RPC is JSONL over pipes; always UTF-8 (Windows default locale is cp1252).
+_PI_SUBPROCESS_ENCODING = "utf-8"
+_PI_SUBPROCESS_ENCODING_ERRORS = "replace"
+
+_PI_INSTALL_HINT = (
+    "Install the Pi coding agent CLI, then restart the Gradio app:  \n"
+    "`npm install -g @earendil-works/pi-coding-agent`  \n"
+    "On Windows, ensure Node.js/npm are on PATH (or set `AGENT_EXECUTABLE` to the "
+    "full path to `pi.cmd`, e.g. `%APPDATA%\\npm\\pi.cmd`).  \n"
+    "Docker users: run the Pi UI via `docker compose` (`pi-agent` service) instead "
+    "of `python gradio_app.py` on the host."
+)
+
+
+def resolve_pi_executable() -> str:
+    """Return a path to the ``pi`` RPC executable (raises ``PiRpcError`` if missing)."""
+    override = os.environ.get("AGENT_EXECUTABLE", "").strip()
+    if override:
+        if os.path.isfile(override) or shutil.which(override):
+            return override
+        raise PiRpcError(
+            f"AGENT_EXECUTABLE is set but not found: `{override}`  \n\n{_PI_INSTALL_HINT}"
+        )
+    for name in ("pi", "pi.cmd"):
+        found = shutil.which(name)
+        if found:
+            return found
+    raise PiRpcError(f"Pi CLI (`pi`) not found on PATH.  \n\n{_PI_INSTALL_HINT}")
+
+
+@dataclass
+class PiStreamEvent:
+    """Structured event from Pi RPC for UI layers."""
+
+    kind: str
+    text: str = ""
+    tool_name: str | None = None
+    tool_call_id: str | None = None
+    tool_args: dict[str, Any] | None = None
+    tool_output: str | None = None
+    is_error: bool = False
+    meta: dict[str, Any] = field(default_factory=dict)
+
+
+def extract_tool_text(payload: dict[str, Any] | None) -> str:
+    if not payload:
+        return ""
+    content = payload.get("content")
+    if content is None and isinstance(payload.get("partialResult"), dict):
+        content = payload["partialResult"].get("content")
+    if content is None and isinstance(payload.get("result"), dict):
+        content = payload["result"].get("content")
+    if not isinstance(content, list):
+        return ""
+    parts: list[str] = []
+    for block in content:
+        if isinstance(block, dict) and block.get("type") == "text":
+            parts.append(str(block.get("text") or ""))
+    return "\n".join(parts).strip()
+
+
+def extract_assistant_display(message: dict[str, Any] | None) -> tuple[str, str]:
+    """Extract visible text and thinking from a partial assistant message."""
+    if not message or message.get("role") != "assistant":
+        return "", ""
+    content = message.get("content")
+    if isinstance(content, str):
+        return content, ""
+    if not isinstance(content, list):
+        return "", ""
+
+    texts: list[str] = []
+    thinkings: list[str] = []
+    for block in content:
+        if isinstance(block, str):
+            if block.strip():
+                texts.append(block)
+            continue
+        if not isinstance(block, dict):
+            continue
+        block_type = block.get("type")
+        if block_type in (None, "text", "output_text"):
+            text = block.get("text") or block.get("content") or ""
+            if text:
+                texts.append(str(text))
+        elif block_type in ("thinking", "reasoning", "thought"):
+            thought = (
+                block.get("thinking")
+                or block.get("text")
+                or block.get("reasoning")
+                or block.get("content")
+                or ""
+            )
+            if thought:
+                thinkings.append(str(thought))
+    return "".join(texts), "".join(thinkings)
+
+
+def assistant_chat_text(visible: str, thinking: str) -> str:
+    """Text to show in the main chat — visible answer, or thinking when Gemini sends only that."""
+    if visible.strip():
+        return visible
+    return thinking
+
+
+def _tool_lines_from_content(content: list[Any]) -> list[str]:
+    tool_lines: list[str] = []
+    for block in content:
+        if not isinstance(block, dict):
+            continue
+        block_type = block.get("type")
+        if block_type not in {"toolCall", "tool_use", "functionCall"}:
+            continue
+        name = str(block.get("name") or block.get("toolName") or "tool")
+        args = block.get("arguments") or block.get("input") or block.get("args")
+        if isinstance(args, str):
+            try:
+                args = json.loads(args)
+            except json.JSONDecodeError:
+                args = {"raw": args}
+        if not isinstance(args, dict):
+            args = {}
+        tool_lines.append(format_tool_chat_line(name, args))
+    return tool_lines
+
+
+def format_tool_chat_line(tool_name: str | None, args: dict[str, Any] | None) -> str:
+    """Render one tool invocation for the chat UI (prose for comment-only bash)."""
+    name = str(tool_name or "tool")
+    lowered = name.lower()
+    if lowered == "bash" and args and args.get("command"):
+        cmd = str(args["command"])
+        if is_bash_commentary_only(cmd):
+            return extract_bash_commentary_text(cmd)
+        commentary, executable = split_bash_commentary_and_command(cmd)
+        if commentary and executable:
+            short = executable[:200] + ("…" if len(executable) > 200 else "")
+            return f"{commentary}\n\n**bash:** `{short}`"
+        if commentary:
+            return commentary
+    detail = format_tool_args(tool_name, args)
+    if detail and detail != name:
+        return f"**{name}:** {detail}"
+    return f"**{name}**"
+
+
+def format_assistant_message_for_chat(message: dict[str, Any]) -> str:
+    """Render one assistant message for the chat UI (visible text or tool calls; no thinking)."""
+    visible, _thinking = extract_assistant_display(message)
+    if visible.strip():
+        return visible
+
+    content = message.get("content")
+    if not isinstance(content, list):
+        return ""
+
+    return "\n".join(_tool_lines_from_content(content))
+
+
+def chat_text_from_assistant_message(message: dict[str, Any] | None) -> str:
+    """Non-thinking chat text from a Pi/Gemini assistant message snapshot."""
+    if not message or message.get("role") != "assistant":
+        return ""
+    return format_assistant_message_for_chat(message)
+
+
+_RATE_LIMIT_MARKERS = (
+    "429",
+    "quota",
+    "rate limit",
+    "rate-limit",
+    "resource_exhausted",
+    "too many requests",
+    "throttlingexception",
+    "throttling",
+    "toomanyrequestsexception",
+    "servicequotaexceeded",
+)
+
+
+def is_rate_limit_error(text: str | None) -> bool:
+    """True when *text* looks like a provider quota or rate-limit failure."""
+    if not text:
+        return False
+    lowered = text.lower()
+    return any(marker in lowered for marker in _RATE_LIMIT_MARKERS)
+
+
+def _strip_rpc_payload_for_debug(obj: Any) -> Any:
+    """
+    Strip large message content from RPC objects for compact debug logging.
+
+    Keeps metadata (id, type, command, success) but removes or truncates
+    actual message/data payloads.
+    """
+    if not isinstance(obj, dict):
+        return obj
+
+    kept_keys = {"type", "id", "command", "success", "error", "stopReason"}
+    result = {k: v for k, v in obj.items() if k in kept_keys}
+
+    # Keep data/result/messages structure without content
+    for key in ("data", "result", "messages", "response"):
+        if key in obj:
+            val = obj[key]
+            if isinstance(val, dict):
+                result[key] = {k: "..." for k in val.keys()}
+            elif isinstance(val, list):
+                result[key] = f"[... {len(val)} items]"
+            else:
+                result[key] = "..."
+
+    return result
+
+
+def last_assistant_turn_error(messages: list[dict[str, Any]]) -> str | None:
+    """Return the latest assistant error in the current user turn, if any."""
+    last_user = -1
+    for index, message in enumerate(messages):
+        if message.get("role") == "user":
+            last_user = index
+
+    turn_messages = messages[last_user + 1 :] if last_user >= 0 else messages
+    for message in reversed(turn_messages):
+        if message.get("role") != "assistant":
+            continue
+        error = message.get("errorMessage")
+        if error:
+            return str(error)
+        if message.get("stopReason") == "error":
+            visible, _ = extract_assistant_display(message)
+            if visible.strip():
+                return visible
+            return "assistant turn failed"
+    return None
+
+
+def assistant_text_since_last_user(messages: list[dict[str, Any]]) -> str:
+    """Combine assistant messages from the latest user turn."""
+    last_user = -1
+    for index, message in enumerate(messages):
+        if message.get("role") == "user":
+            last_user = index
+
+    turn_messages = messages[last_user + 1 :] if last_user >= 0 else messages
+    parts: list[str] = []
+    for message in turn_messages:
+        if message.get("role") != "assistant":
+            continue
+        part = format_assistant_message_for_chat(message)
+        if part.strip():
+            parts.append(part)
+    return "\n\n".join(parts)
+
+
+def partial_message_from_update(event: dict[str, Any]) -> dict[str, Any] | None:
+    delta = event.get("assistantMessageEvent") or {}
+    partial = delta.get("partial")
+    if isinstance(partial, dict):
+        return partial
+    message = event.get("message")
+    if isinstance(message, dict):
+        return message
+    return None
+
+
+def is_bash_commentary_only(command: str) -> bool:
+    """True when a bash tool call contains only shell comments (no executable lines)."""
+    lines = [ln.strip() for ln in command.splitlines() if ln.strip()]
+    if not lines:
+        return False
+    return all(ln.startswith("#") for ln in lines)
+
+
+def extract_bash_commentary_text(command: str) -> str:
+    """Join non-empty ``#`` comment bodies from a bash command into readable prose."""
+    parts: list[str] = []
+    for raw in command.splitlines():
+        stripped = raw.strip()
+        if not stripped.startswith("#"):
+            continue
+        text = stripped.lstrip("#").strip()
+        if text:
+            parts.append(text)
+    return "\n".join(parts)
+
+
+def split_bash_commentary_and_command(command: str) -> tuple[str, str]:
+    """Split ``#`` planning lines from executable shell lines."""
+    comments: list[str] = []
+    commands: list[str] = []
+    for raw in command.splitlines():
+        stripped = raw.strip()
+        if not stripped:
+            continue
+        if stripped.startswith("#"):
+            text = stripped.lstrip("#").strip()
+            if text:
+                comments.append(text)
+        else:
+            commands.append(stripped)
+    return "\n".join(comments), " ↵ ".join(commands)
+
+
+def format_tool_args(tool_name: str | None, args: dict[str, Any] | None) -> str:
+    if not args:
+        return ""
+    name = (tool_name or "").lower()
+    if name == "bash" and args.get("command"):
+        cmd = str(args["command"])
+        if is_bash_commentary_only(cmd):
+            return extract_bash_commentary_text(cmd)
+        _commentary, executable = split_bash_commentary_and_command(cmd)
+        if not executable:
+            return extract_bash_commentary_text(cmd)
+        shown = executable[:240] + ("…" if len(executable) > 240 else "")
+        return f"`{shown}`"
+    if name in {"read", "write", "edit"} and args.get("path"):
+        return f"`{args['path']}`"
+    compact = json.dumps(args, ensure_ascii=False)
+    if len(compact) > 280:
+        compact = compact[:277] + "…"
+    return compact
+
+
+class PiRpcClient:
+    """Drive a long-lived ``pi --mode rpc`` subprocess."""
+
+    # Extension UI dialog methods block Pi until the client replies; auto-cancel
+    # them so a missing UI layer can never wedge the RPC process.
+    _EXTENSION_UI_DIALOG_METHODS = frozenset({"select", "confirm", "input", "editor"})
+
+    def __init__(
+        self,
+        *,
+        cwd: str | None = None,
+        env: dict[str, str] | None = None,
+        pi_args: list[str] | None = None,
+    ) -> None:
+        self._cwd = cwd
+        self._env = env
+        self._pi_args = pi_args or []
+        self._proc: subprocess.Popen[str] | None = None
+        self._write_lock = threading.Lock()
+        self._abort_requested = False
+        self._prompt_stream_depth = 0
+        self._pending_follow_ups = 0
+        self._pending_ui_history: list[dict[str, Any]] = []
+        # Single stdout reader thread demultiplexes the JSONL stream: command
+        # responses go to per-id slots, agent events go to ``_events``. This lets
+        # any thread (e.g. post-task logging) call the client safely while a
+        # prompt stream is active.
+        self._reader_thread: threading.Thread | None = None
+        self._stderr_thread: threading.Thread | None = None
+        self._events: queue.Queue[Any] = queue.Queue()
+        self._pending_lock = threading.Lock()
+        self._pending_responses: dict[str, queue.Queue[Any]] = {}
+        self._stderr_buffer: deque[str] = deque(maxlen=200)
+        self._closing = False
+
+    @property
+    def running(self) -> bool:
+        return self._proc is not None and self._proc.poll() is None
+
+    @property
+    def prompt_stream_active(self) -> bool:
+        """True while :meth:`prompt_events` is consuming the RPC event stream."""
+        return self._prompt_stream_depth > 0
+
+    def start(self) -> None:
+        if self.running:
+            return
+        command = [resolve_pi_executable(), "--mode", "rpc", *self._pi_args]
+        self._closing = False
+        self._abort_requested = False
+        proc = subprocess.Popen(
+            command,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            encoding=_PI_SUBPROCESS_ENCODING,
+            errors=_PI_SUBPROCESS_ENCODING_ERRORS,
+            bufsize=1,
+            cwd=self._cwd,
+            env=self._env,
+        )
+        self._proc = proc
+        # Fresh demux state for this process.
+        self._events = queue.Queue()
+        with self._pending_lock:
+            self._pending_responses = {}
+        self._stderr_buffer = deque(maxlen=200)
+        self._reader_thread = threading.Thread(
+            target=self._reader_loop,
+            args=(proc,),
+            name="pi-rpc-stdout",
+            daemon=True,
+        )
+        self._reader_thread.start()
+        if proc.stderr is not None:
+            self._stderr_thread = threading.Thread(
+                target=self._stderr_loop,
+                args=(proc,),
+                name="pi-rpc-stderr",
+                daemon=True,
+            )
+            self._stderr_thread.start()
+
+    def close(self) -> None:
+        if not self._proc:
+            return
+        self._closing = True
+        proc = self._proc
+        if proc.poll() is None:
+            try:
+                self.abort()
+            except Exception:
+                pass
+            proc.terminate()
+            try:
+                proc.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+        # Process exit makes ``readline`` return EOF; the reader thread then
+        # notifies waiters. Nudge waiters here too in case the threads are slow.
+        self._notify_process_exit()
+        for thread in (self._reader_thread, self._stderr_thread):
+            if (
+                thread is not None
+                and thread.is_alive()
+                and thread is not threading.current_thread()
+            ):
+                thread.join(timeout=2)
+        self._reader_thread = None
+        self._stderr_thread = None
+        self._proc = None
+
+    def _ensure_running(self) -> subprocess.Popen[str]:
+        if not self.running:
+            self.start()
+        assert self._proc is not None
+        return self._proc
+
+    def _recent_stderr(self) -> str:
+        return "\n".join(self._stderr_buffer)
+
+    def _process_exit_error(self) -> PiRpcError:
+        code = self._proc.poll() if self._proc else None
+        err = self._recent_stderr()
+        return PiRpcError(
+            f"Pi RPC process exited (code={code})."
+            + (f" stderr: {err[:500]}" if err else "")
+        )
+
+    def _notify_process_exit(self) -> None:
+        """Unblock every pending response slot and the events queue on exit."""
+        with self._pending_lock:
+            pending = list(self._pending_responses.values())
+            self._pending_responses.clear()
+        for slot in pending:
+            try:
+                slot.put_nowait(_PI_PROCESS_EXIT)
+            except queue.Full:
+                pass
+        try:
+            self._events.put_nowait(_PI_PROCESS_EXIT)
+        except queue.Full:
+            pass
+
+    def _stderr_loop(self, proc: subprocess.Popen[str]) -> None:
+        """Continuously drain stderr into a bounded buffer (prevents pipe deadlock)."""
+        stream = proc.stderr
+        if stream is None:
+            return
+        try:
+            for line in stream:
+                self._stderr_buffer.append(line.rstrip("\r\n"))
+        except (ValueError, OSError):
+            pass
+
+    def _reader_loop(self, proc: subprocess.Popen[str]) -> None:
+        """Read every stdout line and route responses vs. agent events."""
+        stream = proc.stdout
+        if stream is None:
+            self._notify_process_exit()
+            return
+        try:
+            while True:
+                line = stream.readline()
+                if not line:
+                    break
+                line = line.rstrip("\r\n")
+                if not line:
+                    continue
+                try:
+                    message = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                self._dispatch_message(message)
+        except (ValueError, OSError):
+            pass
+        finally:
+            self._notify_process_exit()
+
+    def _dispatch_message(self, message: Any) -> None:
+        if not isinstance(message, dict):
+            return
+        if os.environ.get("AGENT_RPC_DEBUG", "").strip() == "1":
+            try:
+                stripped = _strip_rpc_payload_for_debug(message)
+                sys.stderr.write(
+                    "Pi RPC recv: " + json.dumps(stripped, ensure_ascii=False) + "\n"
+                )
+                sys.stderr.flush()
+            except Exception:
+                pass
+        msg_type = message.get("type")
+        if msg_type == "response":
+            req_id = message.get("id")
+            slot: queue.Queue[Any] | None = None
+            if req_id is not None:
+                with self._pending_lock:
+                    slot = self._pending_responses.pop(str(req_id), None)
+            if slot is not None:
+                try:
+                    slot.put_nowait(message)
+                except queue.Full:
+                    pass
+            return
+        if msg_type == "extension_ui_request":
+            self._auto_reply_extension_ui(message)
+            return
+        # Agent event — consumed by the active ``prompt_events`` stream.
+        self._events.put(message)
+
+    def _auto_reply_extension_ui(self, message: dict[str, Any]) -> None:
+        method = message.get("method")
+        req_id = message.get("id")
+        if req_id is None or method not in self._EXTENSION_UI_DIALOG_METHODS:
+            return
+        try:
+            self._write_command(
+                {"type": "extension_ui_response", "id": req_id, "cancelled": True}
+            )
+        except (OSError, PiRpcError):
+            pass
+
+    def _write_command(self, command: dict[str, Any]) -> None:
+        proc = self._ensure_running()
+        assert proc.stdin is not None
+        if os.environ.get("AGENT_RPC_DEBUG", "").strip() == "1":
+            try:
+                stripped = _strip_rpc_payload_for_debug(command)
+                sys.stderr.write(
+                    "Pi RPC send: " + json.dumps(stripped, ensure_ascii=False) + "\n"
+                )
+                sys.stderr.flush()
+            except Exception:
+                pass
+        with self._write_lock:
+            proc.stdin.write(json.dumps(command) + "\n")
+            proc.stdin.flush()
+
+    def _send_command(
+        self,
+        command: dict[str, Any],
+        *,
+        wait_response: bool = True,
+    ) -> dict[str, Any] | None:
+        req_id = str(command.setdefault("id", str(uuid.uuid4())))
+        if not wait_response:
+            self._write_command(command)
+            return None
+        slot: queue.Queue[Any] = queue.Queue(maxsize=1)
+        with self._pending_lock:
+            self._pending_responses[req_id] = slot
+        try:
+            self._write_command(command)
+        except Exception:
+            with self._pending_lock:
+                self._pending_responses.pop(req_id, None)
+            raise
+        result = slot.get()
+        if result is _PI_PROCESS_EXIT:
+            raise self._process_exit_error()
+        if not result.get("success", False):
+            error = result.get("error") or result.get("message") or "command failed"
+            raise PiRpcError(str(error))
+        return result
+
+    def abort(self) -> None:
+        """Request abort without reading stdout (the active stream consumer drains events)."""
+        if not self.running:
+            return
+        self._abort_requested = True
+        try:
+            self._send_command({"type": "abort"}, wait_response=False)
+        except OSError:
+            pass
+
+    def stage_ui_chat_notice(self, label: str, message: str) -> None:
+        """Stage user/assistant chat rows for the active prompt stream to merge on yield."""
+        text = message.strip()
+        if not text:
+            return
+        self._pending_ui_history.append(
+            {"role": "user", "content": f"_**{label}:**_ {text}"}
+        )
+        self._pending_ui_history.append({"role": "assistant", "content": ""})
+
+    def drain_pending_ui_history(self) -> list[dict[str, Any]]:
+        """Return and clear UI chat rows staged by :meth:`stage_ui_chat_notice`."""
+        pending = self._pending_ui_history[:]
+        self._pending_ui_history.clear()
+        return pending
+
+    def steer(self, message: str) -> None:
+        """Queue a steering message (delivered after the current tool step completes)."""
+        if not message.strip():
+            return
+        self._send_command(
+            {"type": "steer", "message": message},
+            wait_response=False,
+        )
+
+    def follow_up(self, message: str) -> None:
+        """Queue a follow-up message for when the agent stops."""
+        if not message.strip():
+            return
+        self._pending_follow_ups += 1
+        self._send_command(
+            {"type": "follow_up", "message": message},
+            wait_response=False,
+        )
+
+    @property
+    def abort_requested(self) -> bool:
+        return self._abort_requested
+
+    def clear_abort(self) -> None:
+        self._abort_requested = False
+
+    def new_session(self) -> None:
+        self._send_command({"type": "new_session"})
+
+    def get_state(self) -> dict[str, Any]:
+        response = self._send_command({"type": "get_state"})
+        data = response.get("data") if response else {}
+        return data if isinstance(data, dict) else {}
+
+    def get_messages(self) -> list[dict[str, Any]]:
+        response = self._send_command({"type": "get_messages"})
+        data = response.get("data") if response else {}
+        messages = data.get("messages") if isinstance(data, dict) else []
+        return messages if isinstance(messages, list) else []
+
+    def get_session_stats(self) -> dict[str, Any]:
+        """Token usage and cost totals for the active session (Pi RPC ``get_session_stats``)."""
+        response = self._send_command({"type": "get_session_stats"})
+        data = response.get("data") if response else {}
+        return data if isinstance(data, dict) else {}
+
+    def set_model(self, provider: str, model_id: str) -> dict[str, Any]:
+        response = self._send_command(
+            {
+                "type": "set_model",
+                "provider": provider,
+                "modelId": model_id,
+            }
+        )
+        data = response.get("data") if response else {}
+        return data if isinstance(data, dict) else {}
+
+    def get_available_models(self) -> list[dict[str, Any]]:
+        response = self._send_command({"type": "get_available_models"})
+        data = response.get("data") if response else {}
+        models = data.get("models") if isinstance(data, dict) else []
+        return models if isinstance(models, list) else []
+
+    def restart(self) -> None:
+        self.close()
+        self.start()
+
+    def prompt_events(self, message: str) -> Iterator[PiStreamEvent]:
+        """Send a user message and yield structured events until ``agent_end``."""
+        self._prompt_stream_depth += 1
+        try:
+            yield from self._prompt_events_impl(message)
+        finally:
+            self._prompt_stream_depth = max(0, self._prompt_stream_depth - 1)
+
+    def _drain_events(self) -> None:
+        """Discard stale events left over from a prior stream (single active prompt)."""
+        while True:
+            try:
+                item = self._events.get_nowait()
+            except queue.Empty:
+                return
+            if item is _PI_PROCESS_EXIT:
+                # Preserve the exit signal for the consumer to observe.
+                try:
+                    self._events.put_nowait(_PI_PROCESS_EXIT)
+                except queue.Full:
+                    pass
+                return
+
+    def _prompt_events_impl(self, message: str) -> Iterator[PiStreamEvent]:
+        self.clear_abort()
+        self._drain_events()
+        try:
+            self._send_command({"type": "prompt", "message": message})
+        except PiRpcError as exc:
+            yield PiStreamEvent(kind="error", text=str(exc), is_error=True)
+            return
+
+        yield from self._iter_agent_events()
+
+    def _iter_agent_events(self) -> Iterator[PiStreamEvent]:
+        while True:
+            event = self._events.get()
+            if event is _PI_PROCESS_EXIT:
+                raise self._process_exit_error()
+            event_type = event.get("type")
+
+            if event_type == "agent_start":
+                yield PiStreamEvent(kind="status", text="Agent started…")
+
+            elif event_type == "turn_start":
+                yield PiStreamEvent(kind="status", text="Turn started.")
+
+            elif event_type == "turn_end":
+                yield PiStreamEvent(kind="turn_end", text="Turn finished.")
+
+            elif event_type == "message_update":
+                yield from self._parse_message_update(event)
+
+            elif event_type == "tool_execution_start":
+                tool_name = event.get("toolName")
+                tool_args = (
+                    event.get("args") if isinstance(event.get("args"), dict) else {}
+                )
+                yield PiStreamEvent(
+                    kind="tool_start",
+                    tool_name=str(tool_name) if tool_name else "tool",
+                    tool_call_id=event.get("toolCallId"),
+                    tool_args=tool_args,
+                    text=format_tool_args(
+                        str(tool_name) if tool_name else None,
+                        tool_args,
+                    ),
+                )
+
+            elif event_type == "tool_execution_update":
+                output = extract_tool_text(event)
+                yield PiStreamEvent(
+                    kind="tool_update",
+                    tool_name=event.get("toolName"),
+                    tool_call_id=event.get("toolCallId"),
+                    tool_output=output,
+                )
+
+            elif event_type == "tool_execution_end":
+                result = (
+                    event.get("result") if isinstance(event.get("result"), dict) else {}
+                )
+                output = extract_tool_text(result)
+                yield PiStreamEvent(
+                    kind="tool_end",
+                    tool_name=event.get("toolName"),
+                    tool_call_id=event.get("toolCallId"),
+                    tool_output=output,
+                    is_error=bool(event.get("isError")),
+                )
+
+            elif event_type == "queue_update":
+                steering = event.get("steering") or []
+                follow_up = event.get("followUp") or []
+                if steering or follow_up:
+                    yield PiStreamEvent(
+                        kind="queue_update",
+                        meta={"steering": steering, "follow_up": follow_up},
+                    )
+
+            elif event_type == "compaction_start":
+                reason = event.get("reason") or "context limit"
+                yield PiStreamEvent(
+                    kind="compaction_start",
+                    text=(
+                        "Session context is being summarised to stay within the model "
+                        f"limit ({reason}). Older messages and tool output are compressed; "
+                        "recent turns are kept."
+                    ),
+                    meta={"reason": reason, "phase": "start"},
+                )
+
+            elif event_type == "compaction_end":
+                if event.get("aborted"):
+                    text = "Context compaction was aborted."
+                    yield PiStreamEvent(
+                        kind="compaction_end",
+                        text=text,
+                        meta={"phase": "end", "aborted": True},
+                    )
+                elif event.get("errorMessage"):
+                    text = f"Context compaction failed: {event['errorMessage']}"
+                    yield PiStreamEvent(kind="error", text=text, is_error=True)
+                    continue
+                elif event.get("willRetry"):
+                    text = (
+                        "Context compaction finished — retrying the prompt with a "
+                        "shorter history."
+                    )
+                    yield PiStreamEvent(
+                        kind="compaction_end",
+                        text=text,
+                        meta={"phase": "end", "willRetry": True},
+                    )
+                else:
+                    tokens = (event.get("result") or {}).get("tokensBefore")
+                    if isinstance(tokens, int):
+                        text = (
+                            f"Context compaction finished ({tokens:,} tokens before "
+                            "summarisation). The agent continues with a shorter context."
+                        )
+                    else:
+                        text = (
+                            "Context compaction finished. The agent continues with a "
+                            "shorter context."
+                        )
+                    yield PiStreamEvent(
+                        kind="compaction_end",
+                        text=text,
+                        meta={"phase": "end", "tokensBefore": tokens},
+                    )
+
+            elif event_type == "auto_retry_start":
+                attempt = event.get("attempt")
+                max_attempts = event.get("maxAttempts")
+                delay_ms = event.get("delayMs")
+                msg = event.get("errorMessage") or "transient error"
+                yield PiStreamEvent(
+                    kind="status",
+                    text=(
+                        f"Auto-retry {attempt}/{max_attempts} in {delay_ms}ms "
+                        f"({str(msg)[:120]})"
+                    ),
+                    meta=event,
+                )
+
+            elif event_type == "auto_retry_end":
+                if event.get("success"):
+                    yield PiStreamEvent(
+                        kind="status",
+                        text=f"Auto-retry succeeded on attempt {event.get('attempt')}.",
+                    )
+                else:
+                    yield PiStreamEvent(
+                        kind="error",
+                        text=f"Auto-retry failed: {event.get('finalError', 'unknown error')}",
+                        is_error=True,
+                    )
+
+            elif event_type == "extension_error":
+                yield PiStreamEvent(
+                    kind="error",
+                    text=str(event.get("error") or "extension error"),
+                    is_error=True,
+                )
+
+            elif event_type == "agent_end":
+                # Pi delivers queued ``follow_up`` messages after ``agent_end`` and
+                # continues streaming; do not stop the stdout consumer until they run.
+                if self._pending_follow_ups > 0:
+                    self._pending_follow_ups -= 1
+                    yield PiStreamEvent(
+                        kind="status",
+                        text="Follow-up queued — continuing…",
+                    )
+                    continue
+                aborted = self._abort_requested
+                self.clear_abort()
+                yield PiStreamEvent(
+                    kind="done",
+                    text="Agent aborted." if aborted else "Agent finished.",
+                )
+                return
+
+    def _parse_message_update(self, event: dict[str, Any]) -> Iterator[PiStreamEvent]:
+        delta = event.get("assistantMessageEvent") or {}
+        delta_type = delta.get("type")
+        partial = partial_message_from_update(event)
+        if partial is not None:
+            visible, thinking = extract_assistant_display(partial)
+            if visible.strip():
+                yield PiStreamEvent(kind="text_snapshot", text=visible)
+            elif chat_text := chat_text_from_assistant_message(partial):
+                yield PiStreamEvent(kind="text_snapshot", text=chat_text)
+            if thinking.strip():
+                yield PiStreamEvent(kind="thinking_snapshot", text=thinking)
+
+        if delta_type == "text_delta":
+            chunk = delta.get("delta") or ""
+            if chunk:
+                yield PiStreamEvent(kind="text_delta", text=chunk)
+
+        elif delta_type == "thinking_delta":
+            chunk = delta.get("delta") or ""
+            if chunk:
+                yield PiStreamEvent(kind="thinking_delta", text=chunk)
+
+        elif delta_type == "toolcall_start":
+            tool_call = delta.get("toolCall") or {}
+            tool_name = tool_call.get("name") or delta.get("toolName") or "tool"
+            tool_args = tool_call.get("arguments")
+            if isinstance(tool_args, str):
+                try:
+                    tool_args = json.loads(tool_args)
+                except json.JSONDecodeError:
+                    tool_args = {"raw": tool_args}
+            if not isinstance(tool_args, dict):
+                tool_args = {}
+            chat_line = format_tool_chat_line(str(tool_name), tool_args)
+            yield PiStreamEvent(kind="text_snapshot", text=chat_line)
+
+        elif delta_type == "error":
+            yield PiStreamEvent(
+                kind="error",
+                text=str(
+                    delta.get("message") or delta.get("error") or "generation error"
+                ),
+                is_error=True,
+            )
+
+    def prompt_stream(
+        self, message: str, *, show_tool_status: bool = True
+    ) -> Iterator[str]:
+        """Backward-compatible text stream (assistant visible text + optional tool status)."""
+        for event in self.prompt_events(message):
+            if event.kind == "text_delta":
+                yield event.text
+            elif show_tool_status and event.kind == "tool_start":
+                yield f"\n\n_[Running {event.tool_name}…]_\n"
+            elif event.kind == "error":
+                yield f"\n\n**Error:** {event.text}\n"
+
+
+def start_pi_prompt_event_worker(
+    client: PiRpcClient,
+    event_queue: queue.Queue[Any],
+    prompt: str,
+) -> None:
+    """Run ``client.prompt_events`` on a background thread, feeding *event_queue*."""
+
+    def _worker() -> None:
+        try:
+            for event in client.prompt_events(prompt):
+                event_queue.put(event)
+        except Exception as exc:
+            event_queue.put(PiStreamEvent(kind="error", text=str(exc), is_error=True))
+        finally:
+            event_queue.put(None)
+
+    threading.Thread(target=_worker, daemon=True).start()
+
+
+def default_client(session_hash: str | None = None) -> PiRpcClient:
+    from pi_agent_config import configure_aws_credentials, configure_pi_coding_agent_env
+    from pi_workspace_skills import ensure_workspace_skills, pi_rpc_args, pi_rpc_cwd
+
+    configure_aws_credentials()
+    configure_pi_coding_agent_env()
+    ensure_workspace_skills()
+    env = os.environ.copy()
+    env.setdefault("HOME", os.path.expanduser("~"))
+    env.setdefault("PYTHONUTF8", "1")
+    env.setdefault("PYTHONIOENCODING", "utf-8")
+    from session_workspace import workspace_base_dir
+
+    env.setdefault("AGENT_WORKSPACE_DIR", str(workspace_base_dir()))
+    if not env.get("GEMINI_API_KEY") and env.get("GOOGLE_API_KEY"):
+        env["GEMINI_API_KEY"] = env["GOOGLE_API_KEY"]
+    if not env.get("HF_TOKEN") and env.get("DOC_REDACTION_HF_TOKEN"):
+        env["HF_TOKEN"] = env["DOC_REDACTION_HF_TOKEN"]
+    return PiRpcClient(
+        cwd=pi_rpc_cwd(session_hash),
+        env=env,
+        pi_args=pi_rpc_args(),
+    )

agent-redact/pi/pi_session_usage.py

@@ -0,0 +1,190 @@
+"""Summarize Pi agent LLM token usage for usage-log CSV rows."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from pi_rpc_client import PiRpcClient, PiRpcError  # noqa: E402
+
+try:
+    from agent_runtime import AgentRuntime
+except ImportError:
+    AgentRuntime = PiRpcClient  # type: ignore[misc,assignment]
+
+
+@dataclass(frozen=True)
+class TokenUsageTotals:
+    """Pi session usage (see Pi session-format ``Usage``)."""
+
+    input: int = 0
+    output: int = 0
+    cache_read: int = 0
+    cache_write: int = 0
+
+    @property
+    def llm_input_tokens(self) -> int:
+        """Input-side tokens for the main-app usage log (input + cache)."""
+        return self.input + self.cache_read + self.cache_write
+
+    @property
+    def llm_output_tokens(self) -> int:
+        return self.output
+
+
+def _int_field(raw: Any) -> int:
+    try:
+        return max(0, int(raw or 0))
+    except (TypeError, ValueError):
+        return 0
+
+
+def totals_from_usage_dict(usage: dict[str, Any] | None) -> TokenUsageTotals:
+    if not usage:
+        return TokenUsageTotals()
+    return TokenUsageTotals(
+        input=_int_field(usage.get("input")),
+        output=_int_field(usage.get("output")),
+        cache_read=_int_field(usage.get("cacheRead")),
+        cache_write=_int_field(usage.get("cacheWrite")),
+    )
+
+
+def totals_from_stats_payload(data: dict[str, Any] | None) -> TokenUsageTotals:
+    if not data:
+        return TokenUsageTotals()
+    tokens = data.get("tokens")
+    if isinstance(tokens, dict):
+        return totals_from_usage_dict(tokens)
+    return TokenUsageTotals()
+
+
+def subtract_usage(
+    after: TokenUsageTotals, before: TokenUsageTotals
+) -> TokenUsageTotals:
+    return TokenUsageTotals(
+        input=max(0, after.input - before.input),
+        output=max(0, after.output - before.output),
+        cache_read=max(0, after.cache_read - before.cache_read),
+        cache_write=max(0, after.cache_write - before.cache_write),
+    )
+
+
+def add_usage(left: TokenUsageTotals, right: TokenUsageTotals) -> TokenUsageTotals:
+    return TokenUsageTotals(
+        input=left.input + right.input,
+        output=left.output + right.output,
+        cache_read=left.cache_read + right.cache_read,
+        cache_write=left.cache_write + right.cache_write,
+    )
+
+
+def sum_usage_from_messages(
+    messages: list[dict[str, Any]],
+    *,
+    since_last_user: bool = False,
+) -> TokenUsageTotals:
+    """Sum ``usage`` on assistant messages (optional: only after the last user turn)."""
+    last_user = -1
+    if since_last_user:
+        for index, message in enumerate(messages):
+            if message.get("role") == "user":
+                last_user = index
+        messages = messages[last_user + 1 :] if last_user >= 0 else messages
+
+    total = TokenUsageTotals()
+    for message in messages:
+        if message.get("role") != "assistant":
+            continue
+        usage = message.get("usage")
+        if isinstance(usage, dict):
+            total = add_usage(total, totals_from_usage_dict(usage))
+    return total
+
+
+def sum_usage_from_jsonl(path: Path) -> TokenUsageTotals:
+    """Parse a Pi session JSONL file and sum assistant ``usage`` blocks."""
+    total = TokenUsageTotals()
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError:
+        return total
+    for line in text.splitlines():
+        stripped = line.strip()
+        if not stripped:
+            continue
+        try:
+            entry = json.loads(stripped)
+        except json.JSONDecodeError:
+            continue
+        if entry.get("type") != "message":
+            continue
+        message = entry.get("message")
+        if not isinstance(message, dict) or message.get("role") != "assistant":
+            continue
+        usage = message.get("usage")
+        if isinstance(usage, dict):
+            total = add_usage(total, totals_from_usage_dict(usage))
+    return total
+
+
+def resolve_session_token_usage(client: AgentRuntime | None) -> TokenUsageTotals:
+    """
+    Best-effort session usage from Pi RPC ``get_session_stats``, live messages, or JSONL.
+    """
+    if client is None or not client.running:
+        return TokenUsageTotals()
+
+    try:
+        stats = client.get_session_stats()
+        totals = totals_from_stats_payload(stats)
+        if totals.input or totals.output or totals.cache_read or totals.cache_write:
+            return totals
+    except PiRpcError:
+        pass
+
+    try:
+        messages = client.get_messages()
+        totals = sum_usage_from_messages(messages)
+        if totals.input or totals.output or totals.cache_read or totals.cache_write:
+            return totals
+    except PiRpcError:
+        pass
+
+    from session_logs import pi_session_file_from_client
+
+    session_file = pi_session_file_from_client(client)
+    if session_file is not None:
+        return sum_usage_from_jsonl(session_file)
+    return TokenUsageTotals()
+
+
+def usage_for_completed_turn(
+    client: AgentRuntime | None,
+    baseline: TokenUsageTotals | None,
+) -> TokenUsageTotals:
+    """
+    Tokens consumed by the prompt that just finished.
+
+    Prefers delta from *baseline* (captured before ``prompt_events``). Falls back to
+    summing assistant ``usage`` since the last user message, then whole-session totals.
+    """
+    if client is None or not client.running:
+        return TokenUsageTotals()
+
+    current = resolve_session_token_usage(client)
+    if baseline is not None:
+        delta = subtract_usage(current, baseline)
+        if delta.input or delta.output or delta.cache_read or delta.cache_write:
+            return delta
+
+    try:
+        turn = sum_usage_from_messages(client.get_messages(), since_last_user=True)
+        if turn.input or turn.output or turn.cache_read or turn.cache_write:
+            return turn
+    except PiRpcError:
+        pass
+
+    return current

agent-redact/pi/pi_workspace_skills.py

@@ -0,0 +1,397 @@
+"""Sync doc_redaction skills into the Pi workspace and constrain Pi RPC to that tree."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import stat
+from pathlib import Path
+
+from bootstrap_pi_config import pi_repo_root_path
+
+
+def workspace_base_dir() -> Path:
+    from session_workspace import workspace_base_dir as _base
+
+    return _base()
+
+
+def workspace_pi_dir() -> Path:
+    return workspace_base_dir() / ".pi"
+
+
+def workspace_skills_dir() -> Path:
+    return workspace_pi_dir() / "skills"
+
+
+def workspace_helpers_dir() -> Path:
+    return workspace_pi_dir() / "helpers"
+
+
+def remote_redaction_helper_path() -> Path:
+    """Absolute path to synced ``remote_redaction.py`` (always under workspace base, not session subfolders)."""
+    return workspace_helpers_dir() / "remote_redaction.py"
+
+
+def remote_redaction_helper_module() -> str:
+    return remote_redaction_helper_path().as_posix()
+
+
+def repo_skills_dir() -> Path:
+    return pi_repo_root_path() / "skills"
+
+
+def _env_flag(name: str) -> bool:
+    return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
+
+
+_SKILLS_SKIP_DIR_NAMES = frozenset({"archive_attempts"})
+_SKILLS_SKIP_SUFFIXES = (".b64.txt",)
+_SKILLS_MAX_FILE_BYTES = int(
+    os.environ.get("AGENT_SKILLS_MAX_FILE_BYTES", str(512 * 1024))
+)
+
+
+def _should_skip_skill_relpath(rel: Path, *, size_bytes: int | None = None) -> bool:
+    """Skip archive blobs and other non-skill artifacts during workspace sync."""
+    if any(part in _SKILLS_SKIP_DIR_NAMES for part in rel.parts):
+        return True
+    name_lower = rel.name.lower()
+    if name_lower.endswith(_SKILLS_SKIP_SUFFIXES):
+        return True
+    if size_bytes is not None and size_bytes > _SKILLS_MAX_FILE_BYTES:
+        return True
+    return False
+
+
+def _should_resync(dest: Path, src: Path) -> bool:
+    if _env_flag("AGENT_SKILLS_RESYNC"):
+        return True
+    if not dest.is_dir():
+        return True
+    if not any(dest.iterdir()):
+        return True
+    try:
+        return src.stat().st_mtime > dest.stat().st_mtime
+    except OSError:
+        return True
+
+
+def _copy_tree_item(src: Path, dest: Path) -> None:
+    _copy_tree_item_filtered(src, dest, src_root=src)
+
+
+def _copy_to_workspace(src: Path, dest: Path) -> None:
+    """Copy file content without metadata (copy2/copystat raises EPERM on bind mounts)."""
+    shutil.copyfile(src, dest)
+
+
+def _copy_tree_item_filtered(src: Path, dest: Path, *, src_root: Path) -> None:
+    rel = src.relative_to(src_root)
+    if _should_skip_skill_relpath(rel):
+        return
+    if src.is_file():
+        try:
+            size = src.stat().st_size
+        except OSError:
+            size = None
+        if size is not None and size > _SKILLS_MAX_FILE_BYTES:
+            return
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        if dest.exists():
+            _make_writable(dest)
+        _copy_to_workspace(src, dest)
+        return
+    if dest.exists():
+        for child in sorted(src.iterdir()):
+            _copy_tree_item_filtered(child, dest / child.name, src_root=src_root)
+    else:
+        dest.mkdir(parents=True, exist_ok=True)
+        for child in sorted(src.iterdir()):
+            _copy_tree_item_filtered(child, dest / child.name, src_root=src_root)
+
+
+def _chmod_tree(path: Path, *, writable: bool) -> None:
+    """Set or clear write bits on a file tree (needed for Windows resync)."""
+    try:
+        if path.is_dir():
+            for root, dirs, files in os.walk(path):
+                root_path = Path(root)
+                for name in files:
+                    file_path = root_path / name
+                    mode = file_path.stat().st_mode
+                    file_path.chmod(
+                        (mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
+                        if writable
+                        else (mode & ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH)
+                    )
+                for name in dirs:
+                    dir_path = root_path / name
+                    mode = dir_path.stat().st_mode
+                    dir_path.chmod(
+                        (mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
+                        if writable
+                        else (mode & ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH)
+                    )
+            mode = path.stat().st_mode
+            path.chmod(
+                (mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
+                if writable
+                else (mode & ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH)
+            )
+        else:
+            mode = path.stat().st_mode
+            path.chmod(
+                (mode | stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
+                if writable
+                else (mode & ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH)
+            )
+    except OSError:
+        pass
+
+
+def _make_writable(path: Path) -> None:
+    _chmod_tree(path, writable=True)
+
+
+def _make_readonly(path: Path) -> None:
+    if _env_flag("AGENT_SKILLS_WRITABLE"):
+        return
+    _chmod_tree(path, writable=False)
+
+
+def write_workspace_pi_settings() -> Path:
+    """
+    Project Pi settings under ``{workspace}/.pi/settings.json``.
+
+    Paths in that file resolve relative to ``{workspace}/.pi/`` per Pi docs.
+    """
+    pi_dir = workspace_pi_dir()
+    pi_dir.mkdir(parents=True, exist_ok=True)
+    settings_path = pi_dir / "settings.json"
+    payload = {
+        "skills": ["skills"],
+        "extensions": [],
+        "packages": [],
+        "enableSkillCommands": True,
+    }
+    import json
+
+    settings_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    return settings_path
+
+
+def sync_repo_skills_to_workspace(*, force: bool = False) -> Path:
+    """
+    Copy ``{repo}/skills/`` → ``{workspace}/.pi/skills/`` (read-only for the agent).
+
+    Re-sync when the repo tree is newer or ``AGENT_SKILLS_RESYNC=true``.
+    """
+    src = repo_skills_dir()
+    dest = workspace_skills_dir()
+    workspace_pi_dir().mkdir(parents=True, exist_ok=True)
+
+    if not src.is_dir():
+        dest.mkdir(parents=True, exist_ok=True)
+        write_workspace_pi_settings()
+        return dest
+
+    if force or _should_resync(dest, src):
+        if dest.exists():
+            _make_writable(dest)
+            shutil.rmtree(dest)
+        dest.mkdir(parents=True, exist_ok=True)
+        for item in sorted(src.iterdir()):
+            rel = item.relative_to(src)
+            try:
+                size = item.stat().st_size if item.is_file() else None
+            except OSError:
+                size = None
+            if _should_skip_skill_relpath(rel, size_bytes=size):
+                continue
+            _copy_tree_item_filtered(item, dest / item.name, src_root=src)
+
+    write_workspace_pi_settings()
+    os.environ["AGENT_WORKSPACE_SKILLS_DIR"] = str(dest.resolve())
+    return dest.resolve()
+
+
+def sync_workspace_helpers() -> Path:
+    """
+    Copy Pi redaction helper scripts into ``{workspace}/.pi/helpers/``.
+
+    Keeps ``remote_redaction.py`` inside the workspace boundary on AWS ECS so the
+    agent does not search ``/workspace/doc_redaction/agent-redact/``.
+    """
+    helpers = workspace_helpers_dir()
+    helpers.mkdir(parents=True, exist_ok=True)
+    shared_dir = Path(__file__).resolve().parent.parent / "shared"
+    for name in ("remote_redaction.py", "run_doc_redact.py"):
+        src = shared_dir / name
+        dest = helpers / name
+        if not src.is_file():
+            continue
+        if not dest.is_file() or src.stat().st_mtime > dest.stat().st_mtime:
+            _copy_to_workspace(src, dest)
+    return helpers.resolve()
+
+
+def write_hf_space_deployment_skill(*, force: bool = False) -> Path | None:
+    """
+    Write a deployment-specific skill that overrides Docker URLs in generic skills.
+
+    Only active when ``AGENT_DEPLOYMENT_PROFILE=hf-space``.
+    """
+    try:
+        from pi_agent_config import is_hf_space_profile
+        from redaction_prompt import doc_redaction_gradio_url
+    except ImportError:
+        return None
+    if not is_hf_space_profile():
+        return None
+
+    skills_root = workspace_skills_dir()
+    skills_root.mkdir(parents=True, exist_ok=True)
+    if skills_root.is_dir():
+        _make_writable(skills_root)
+
+    dest_dir = skills_root / "hf-space-deployment"
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    dest = dest_dir / "SKILL.md"
+    url = doc_redaction_gradio_url()
+    helpers = workspace_helpers_dir().as_posix()
+    content = (
+        "# HF Space deployment (read first)\n\n"
+        "This Pi agent runs on **Hugging Face Spaces** with **Gemini** and calls a "
+        "**remote** doc_redaction Space. Generic skills mention Docker URLs for "
+        "local-docker or AWS ECS — **ignore those here**.\n\n"
+        "## Authoritative settings\n\n"
+        "| Setting | Value |\n"
+        "|---------|--------|\n"
+        f"| **doc_redaction URL** | `{url}` **only** |\n"
+        "| **Auth** | `HF_TOKEN` (Space secret; already in Pi subprocess env) |\n"
+        f"| **Helper module** | `{helpers}/remote_redaction.py` |\n\n"
+        "## One-shot CLI (preferred over writing ``run_redact.py``)\n\n"
+        f"```bash\n"
+        f"python3 {helpers}/run_doc_redact.py \\\n"
+        f'  --pdf "<session-folder>/document.pdf" \\\n'
+        f'  --dest "<session-folder>/redact/document.pdf/output_redact/" \\\n'
+        f'  --ocr-method "Local model - selectable text" \\\n'
+        f'  --pii-method "Local"\n'
+        f"```\n\n"
+        "## Minimal Python (only if the CLI is insufficient)\n\n"
+        "```python\n"
+        "import importlib.util\n"
+        "import sys\n"
+        f'helper = "{helpers}/remote_redaction.py"\n'
+        'spec = importlib.util.spec_from_file_location("remote_redaction", helper)\n'
+        "mod = importlib.util.module_from_spec(spec)\n"
+        'sys.modules["remote_redaction"] = mod\n'
+        "spec.loader.exec_module(mod)\n"
+        "from gradio_client import handle_file\n\n"
+        f"client = mod.make_redaction_client()  # URL: {url}\n"
+        'pdf = "<your-session-folder>/document.pdf"\n'
+        "result = client.predict(\n"
+        '    api_name="/doc_redact",\n'
+        "    document_file=handle_file(pdf),\n"
+        ")\n"
+        'paths = mod.resolve_redaction_output_paths(result, document_stem="document")\n'
+        'mod.fetch_redaction_files(paths, "<your-session-folder>/redact/document/output_redact/")\n'
+        "```\n\n"
+        "## Rules\n\n"
+        f"- **Helper path is shared:** `{helpers}/remote_redaction.py` lives under the "
+        f"workspace root `{workspace_base_dir().as_posix()}/`, **not** under your session "
+        f"subfolder's `.pi/` tree.\n"
+        f"- Call `/doc_redact` via `{helpers}/run_doc_redact.py` or `make_redaction_client()`.\n"
+        "- **Do not** create `run_redact.py`, `run_redact_fixed.py`, or duplicate helpers in your session folder.\n"
+        "- **Do not** call `Client(...)` or `view_api()` in a loop from bash — each call hits HF rate limits. "
+        "Use the CLI once, or one `make_redaction_client()` (cached + retries).\n"
+        "- **Do not** pass `base_url=` manually — `make_redaction_client()` reads "
+        f"`DOC_REDACTION_GRADIO_URL` (`{url}`).\n"
+        "- **Do not** use `host.docker.internal`, `localhost`, `redaction:7861`, or probe "
+        "alternate URLs.\n"
+        "- **Do not** rewrite or duplicate `remote_redaction.py` — use the synced helper.\n"
+        "- On `TooManyRequestsError`, wait at least 60s and retry **once** via the CLI — "
+        "do not spawn repeated `python3 -c` Client probes.\n"
+        "- Write status updates as **normal assistant text**, not bash `#` comments.\n"
+        "- After `/doc_redact`, download outputs with `fetch_redaction_files` into your "
+        "session `output_redact/` folder.\n\n"
+        "Then read `/skill:doc-redaction-app` and `/skill:doc-redaction-modifications` "
+        "for workflow steps, substituting the URL above wherever examples show Docker hosts.\n"
+    )
+    if force or not dest.is_file() or dest.read_text(encoding="utf-8") != content:
+        dest.write_text(content, encoding="utf-8")
+    return dest
+
+
+def ensure_workspace_skills(*, force: bool = False) -> Path:
+    """Idempotent sync used at app startup and before Pi RPC starts."""
+    dest = sync_repo_skills_to_workspace(force=force)
+    sync_workspace_helpers()
+    write_hf_space_deployment_skill(force=force)
+    if dest.is_dir():
+        _make_readonly(dest)
+    return dest
+
+
+def partnership_template_in_workspace() -> Path | None:
+    path = workspace_skills_dir() / "Example prompt partnership.txt"
+    return path if path.is_file() else None
+
+
+def pi_rpc_cwd(session_hash: str | None = None) -> str:
+    """Subprocess cwd for ``pi --mode rpc`` (session subfolder when enabled)."""
+    from session_workspace import session_workspace_dir, session_workspace_enabled
+
+    base = workspace_base_dir()
+    if session_hash and session_hash.strip() and session_workspace_enabled():
+        return str(session_workspace_dir(session_hash))
+    return str(base)
+
+
+def pi_rpc_args() -> list[str]:
+    """Load only workspace skills; do not discover repo ``skills/`` via ancestors."""
+    skills_dir = ensure_workspace_skills()
+    return ["--no-skills", "--skill", str(skills_dir)]
+
+
+def workspace_boundary_prefix(session_hash: str | None = None) -> str:
+    """Extra prompt text: workspace root, skills path, and path rules."""
+    base = workspace_base_dir().as_posix().rstrip("/")
+    skills = workspace_skills_dir().as_posix()
+    from session_workspace import session_workspace_dir, session_workspace_enabled
+
+    if session_hash and session_hash.strip() and session_workspace_enabled():
+        root = session_workspace_dir(session_hash).as_posix().rstrip("/")
+        scope = f"your session folder `{root}/`"
+    else:
+        root = base
+        scope = f"the workspace `{base}/`"
+
+    hf_note = ""
+    try:
+        from pi_agent_config import is_hf_space_profile
+        from redaction_prompt import doc_redaction_gradio_url
+
+        if is_hf_space_profile():
+            helpers = remote_redaction_helper_module()
+            hf_note = (
+                f"**HF Space redaction backend:** use `{doc_redaction_gradio_url()}` only "
+                f"(see `/skill:hf-space-deployment`). Import helpers from `{helpers}` "
+                f"(workspace base — not `{root}/.pi/helpers/`). Do not use Docker host "
+                "URLs from other skills. Write user-facing progress as normal chat text, "
+                "not bash comments.\n\n"
+            )
+    except ImportError:
+        pass
+
+    return (
+        f"**Workspace boundary (mandatory):** work only under `{base}/`. "
+        f"Your active directory is {scope}. "
+        f"Do not read, write, or run shell commands targeting paths outside `{base}/` "
+        f"(including the git checkout and `agent-redact/`). "
+        f"**Skills (read-only):** doc_redaction skills are synced to `{skills}/`. "
+        f"Use `/skill:doc-redaction-app`, `/skill:doc-redact-page-review`, etc. "
+        f"Do not edit files under `{skills}/`.\n\n"
+        f"{hf_note}"
+    )

agent-redact/redaction_langgraph/__init__.py

@@ -0,0 +1 @@
+"""LangGraph-based redaction orchestration (curated tools, no shell)."""