Sync: Agentic app title change. lint check

.coveragerc

@@ -0,0 +1,56 @@
+[run]
+source = .
+omit = 
+    */tests/*
+    */test/*
+    */__pycache__/*
+    */venv/*
+    */env/*
+    */build/*
+    */dist/*
+    */cdk/*
+    */docs/*
+    */example_data/*
+    */examples/*
+    */feedback/*
+    */logs/*
+    */old_code/*
+    */output/*
+    */tmp/*
+    */usage/*
+    */tld/*
+    */tesseract/*
+    */poppler/*
+    config*.py
+    setup.py
+    lambda_entrypoint.py
+    entrypoint.sh
+    cli_redact.py
+    load_dynamo_logs.py
+    load_s3_logs.py
+    *.spec
+    Dockerfile
+    *.qmd
+    *.md
+    *.txt
+    *.yml
+    *.yaml
+    *.json
+    *.csv
+    *.env
+    *.bat
+    *.ps1
+    *.sh
+
+[report]
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    if self.debug:
+    if settings.DEBUG
+    raise AssertionError
+    raise NotImplementedError
+    if 0:
+    if __name__ == .__main__.:
+    class .*\bProtocol\):
+    @(abc\.)?abstractmethod

.dockerignore

@@ -0,0 +1,52 @@
+*.url
+*.ipynb
+*.pyc
+*.qmd
+_quarto.yml
+quarto_site/*
+src/*
+redaction_deps/*
+.venv/*
+examples/*
+processing/*
+tools/__pycache__/*
+old_code/*
+tesseract/*
+poppler/*
+build/*
+dist/*
+docs/*
+.pi/*
+build_deps/*
+user_guide/*
+_extensions/*
+workspace/*
+doc_redaction.egg-info/*
+.venv_pypi_test/*
+cdk/config/*
+tld/*
+cdk/config/*
+cdk/cdk.out/*
+cdk/archive/*
+cdk.json
+cdk.context.json
+.quarto/*
+logs/
+output/
+input/
+feedback/
+config/
+usage/
+test/config/*
+test/feedback/*
+test/input/*
+test/logs/*
+test/output/*
+test/tmp/*
+test/usage/*
+.ruff_cache/*
+model_cache/*
+sanitized_file/*
+src/doc_redaction.egg-info/*
+docker_compose/*
+skills/example_prompts/*

.gitattributes

@@ -0,0 +1,9 @@
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.sh text eol=lf
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.xls filter=lfs diff=lfs merge=lfs -text
+*.xlsx filter=lfs diff=lfs merge=lfs -text
+*.docx filter=lfs diff=lfs merge=lfs -text
+*.doc filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.ico filter=lfs diff=lfs merge=lfs -text

.github/scripts/setup_test_data.py

@@ -0,0 +1,320 @@
+#!/usr/bin/env python3
+"""
+Setup script for GitHub Actions test data.
+Creates dummy test files when example data is not available.
+"""
+
+import os
+import sys
+
+import pandas as pd
+
+
+def create_directories():
+    """Create necessary directories."""
+    dirs = ["doc_redaction/example_data", "doc_redaction/example_data/example_outputs"]
+
+    for dir_path in dirs:
+        os.makedirs(dir_path, exist_ok=True)
+        print(f"Created directory: {dir_path}")
+
+
+def create_dummy_pdf():
+    """Create dummy PDFs for testing."""
+
+    # Install reportlab if not available
+    try:
+        from reportlab.lib.pagesizes import letter
+        from reportlab.pdfgen import canvas
+    except ImportError:
+        import subprocess
+
+        subprocess.check_call(["pip", "install", "reportlab"])
+        from reportlab.lib.pagesizes import letter
+        from reportlab.pdfgen import canvas
+
+    try:
+        # Create the main test PDF
+        pdf_path = "doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf"
+        print(f"Creating PDF: {pdf_path}")
+        print(f"Directory exists: {os.path.exists('doc_redaction/example_data')}")
+
+        c = canvas.Canvas(pdf_path, pagesize=letter)
+        c.drawString(100, 750, "This is a test document for redaction testing.")
+        c.drawString(100, 700, "Email: test@example.com")
+        c.drawString(100, 650, "Phone: 123-456-7890")
+        c.drawString(100, 600, "Name: John Doe")
+        c.drawString(100, 550, "Address: 123 Test Street, Test City, TC 12345")
+        c.showPage()
+
+        # Add second page
+        c.drawString(100, 750, "Second page content")
+        c.drawString(100, 700, "More test data: jane.doe@example.com")
+        c.drawString(100, 650, "Another phone: 987-654-3210")
+        c.save()
+
+        print(f"Created dummy PDF: {pdf_path}")
+
+        # Create Partnership Agreement Toolkit PDF
+        partnership_pdf_path = (
+            "doc_redaction/example_data/Partnership-Agreement-Toolkit_0_0.pdf"
+        )
+        print(f"Creating PDF: {partnership_pdf_path}")
+        c = canvas.Canvas(partnership_pdf_path, pagesize=letter)
+        c.drawString(100, 750, "Partnership Agreement Toolkit")
+        c.drawString(100, 700, "This is a test partnership agreement document.")
+        c.drawString(100, 650, "Contact: partnership@example.com")
+        c.drawString(100, 600, "Phone: (555) 123-4567")
+        c.drawString(100, 550, "Address: 123 Partnership Street, City, State 12345")
+        c.showPage()
+
+        # Add second page
+        c.drawString(100, 750, "Page 2 - Partnership Details")
+        c.drawString(100, 700, "More partnership information here.")
+        c.drawString(100, 650, "Contact: info@partnership.org")
+        c.showPage()
+
+        # Add third page
+        c.drawString(100, 750, "Page 3 - Terms and Conditions")
+        c.drawString(100, 700, "Terms and conditions content.")
+        c.drawString(100, 650, "Legal contact: legal@partnership.org")
+        c.save()
+
+        print(f"Created dummy PDF: {partnership_pdf_path}")
+
+        # Create Graduate Job Cover Letter PDF
+        cover_letter_pdf_path = (
+            "doc_redaction/example_data/graduate-job-example-cover-letter.pdf"
+        )
+        print(f"Creating PDF: {cover_letter_pdf_path}")
+        c = canvas.Canvas(cover_letter_pdf_path, pagesize=letter)
+        c.drawString(100, 750, "Cover Letter Example")
+        c.drawString(100, 700, "Dear Hiring Manager,")
+        c.drawString(100, 650, "I am writing to apply for the position.")
+        c.drawString(100, 600, "Contact: applicant@example.com")
+        c.drawString(100, 550, "Phone: (555) 987-6543")
+        c.drawString(100, 500, "Address: 456 Job Street, Employment City, EC 54321")
+        c.drawString(100, 450, "Sincerely,")
+        c.drawString(100, 400, "John Applicant")
+        c.save()
+
+        print(f"Created dummy PDF: {cover_letter_pdf_path}")
+
+    except ImportError:
+        print("ReportLab not available, skipping PDF creation")
+        # Create simple text files instead
+        with open(
+            "doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf",
+            "w",
+        ) as f:
+            f.write("This is a dummy PDF file for testing")
+
+        with open(
+            "doc_redaction/example_data/Partnership-Agreement-Toolkit_0_0.pdf",
+            "w",
+        ) as f:
+            f.write("This is a dummy Partnership Agreement PDF file for testing")
+
+        with open(
+            "doc_redaction/example_data/graduate-job-example-cover-letter.pdf",
+            "w",
+        ) as f:
+            f.write("This is a dummy cover letter PDF file for testing")
+
+        print("Created dummy text files instead of PDFs")
+
+
+def create_dummy_csv():
+    """Create dummy CSV files for testing."""
+    # Main CSV
+    csv_data = {
+        "Case Note": [
+            "Client visited for consultation regarding housing issues",
+            "Follow-up appointment scheduled for next week",
+            "Documentation submitted for review",
+        ],
+        "Client": ["John Smith", "Jane Doe", "Bob Johnson"],
+        "Date": ["2024-01-15", "2024-01-16", "2024-01-17"],
+    }
+    df = pd.DataFrame(csv_data)
+    df.to_csv("doc_redaction/example_data/combined_case_notes.csv", index=False)
+    print("Created dummy CSV: doc_redaction/example_data/combined_case_notes.csv")
+
+    # Lambeth CSV
+    lambeth_data = {
+        "text": [
+            "Lambeth 2030 vision document content",
+            "Our Future Our Lambeth strategic plan",
+            "Community engagement and development",
+        ],
+        "page": [1, 2, 3],
+    }
+    df_lambeth = pd.DataFrame(lambeth_data)
+    df_lambeth.to_csv(
+        "doc_redaction/example_data/Lambeth_2030-Our_Future_Our_Lambeth.pdf.csv",
+        index=False,
+    )
+    print(
+        "Created dummy CSV: doc_redaction/example_data/Lambeth_2030-Our_Future_Our_Lambeth.pdf.csv"
+    )
+
+
+def create_dummy_word_doc():
+    """Create dummy Word document."""
+    try:
+        from docx import Document
+
+        doc = Document()
+        doc.add_heading("Test Document for Redaction", 0)
+        doc.add_paragraph("This is a test document for redaction testing.")
+        doc.add_paragraph("Contact Information:")
+        doc.add_paragraph("Email: test@example.com")
+        doc.add_paragraph("Phone: 123-456-7890")
+        doc.add_paragraph("Name: John Doe")
+        doc.add_paragraph("Address: 123 Test Street, Test City, TC 12345")
+
+        doc.save(
+            "doc_redaction/example_data/Bold minimalist professional cover letter.docx"
+        )
+        print("Created dummy Word document")
+
+    except ImportError:
+        print("python-docx not available, skipping Word document creation")
+
+
+def create_allow_deny_lists():
+    """Create dummy allow/deny lists."""
+    # Allow lists
+    allow_data = {"word": ["test", "example", "document"]}
+    pd.DataFrame(allow_data).to_csv(
+        "doc_redaction/example_data/test_allow_list_graduate.csv", index=False
+    )
+    pd.DataFrame(allow_data).to_csv(
+        "doc_redaction/example_data/test_allow_list_partnership.csv", index=False
+    )
+    print("Created allow lists")
+
+    # Deny lists
+    deny_data = {"word": ["sensitive", "confidential", "private"]}
+    pd.DataFrame(deny_data).to_csv(
+        "doc_redaction/example_data/partnership_toolkit_redact_custom_deny_list.csv",
+        index=False,
+    )
+    pd.DataFrame(deny_data).to_csv(
+        "doc_redaction/example_data/Partnership-Agreement-Toolkit_test_deny_list_para_single_spell.csv",
+        index=False,
+    )
+    print("Created deny lists")
+
+    # Whole page redaction list
+    page_data = {"page": [1, 2]}
+    pd.DataFrame(page_data).to_csv(
+        "doc_redaction/example_data/partnership_toolkit_redact_some_pages.csv",
+        index=False,
+    )
+    print("Created whole page redaction list")
+
+
+def create_ocr_output():
+    """Create dummy OCR output CSV."""
+    ocr_data = {
+        "page": [1, 2, 3],
+        "text": [
+            "This is page 1 content with some text",
+            "This is page 2 content with different text",
+            "This is page 3 content with more text",
+        ],
+        "left": [0.1, 0.3, 0.5],
+        "top": [0.95, 0.92, 0.88],
+        "width": [0.05, 0.02, 0.02],
+        "height": [0.01, 0.02, 0.02],
+        "line": [1, 2, 3],
+    }
+    df = pd.DataFrame(ocr_data)
+    df.to_csv(
+        "doc_redaction/example_data/example_outputs/doubled_output_joined.pdf_ocr_output.csv",
+        index=False,
+    )
+    print("Created dummy OCR output CSV")
+
+
+def create_dummy_image():
+    """Create dummy image for testing."""
+    try:
+        from PIL import Image, ImageDraw, ImageFont
+
+        img = Image.new("RGB", (800, 600), color="white")
+        draw = ImageDraw.Draw(img)
+
+        # Try to use a system font
+        try:
+            font = ImageFont.truetype(
+                "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 20
+            )
+        except Exception as e:
+            print(f"Error loading DejaVuSans font: {e}")
+            try:
+                font = ImageFont.truetype("/System/Library/Fonts/Arial.ttf", 20)
+            except Exception as e:
+                print(f"Error loading Arial font: {e}")
+                font = ImageFont.load_default()
+
+        # Add text to image
+        draw.text((50, 50), "Test Document for Redaction", fill="black", font=font)
+        draw.text((50, 100), "Email: test@example.com", fill="black", font=font)
+        draw.text((50, 150), "Phone: 123-456-7890", fill="black", font=font)
+        draw.text((50, 200), "Name: John Doe", fill="black", font=font)
+        draw.text((50, 250), "Address: 123 Test Street", fill="black", font=font)
+
+        img.save("doc_redaction/example_data/example_complaint_letter.jpg")
+        print("Created dummy image")
+
+    except ImportError:
+        print("PIL not available, skipping image creation")
+
+
+def main():
+    """Main setup function."""
+    print("Setting up test data for GitHub Actions...")
+    print(f"Current working directory: {os.getcwd()}")
+    print(f"Python version: {sys.version}")
+
+    create_directories()
+    create_dummy_pdf()
+    create_dummy_csv()
+    create_dummy_word_doc()
+    create_allow_deny_lists()
+    create_ocr_output()
+    create_dummy_image()
+
+    print("\nTest data setup complete!")
+    print("Created files:")
+    for root, dirs, files in os.walk("doc_redaction/example_data"):
+        for file in files:
+            file_path = os.path.join(root, file)
+            print(f"  {file_path}")
+            # Verify the file exists and has content
+            if os.path.exists(file_path):
+                file_size = os.path.getsize(file_path)
+                print(f"    Size: {file_size} bytes")
+            else:
+                print("    WARNING: File does not exist!")
+
+    # Verify critical files exist
+    critical_files = [
+        "doc_redaction/example_data/Partnership-Agreement-Toolkit_0_0.pdf",
+        "doc_redaction/example_data/graduate-job-example-cover-letter.pdf",
+        "doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf",
+    ]
+
+    print("\nVerifying critical test files:")
+    for file_path in critical_files:
+        if os.path.exists(file_path):
+            file_size = os.path.getsize(file_path)
+            print(f"✅ {file_path} exists ({file_size} bytes)")
+        else:
+            print(f"❌ {file_path} MISSING!")
+
+
+if __name__ == "__main__":
+    main()

.github/workflow_README.md

@@ -0,0 +1,183 @@
+# GitHub Actions CI/CD Setup
+
+This directory contains GitHub Actions workflows for automated testing of the CLI redaction application.
+
+## Workflows Overview
+
+### 1. **Simple Test Run** (`.github/workflows/simple-test.yml`)
+- **Purpose**: Basic test execution
+- **Triggers**: Push to main/dev, Pull requests
+- **OS**: Ubuntu Latest
+- **Python**: 3.11
+- **Features**: 
+  - Installs system dependencies
+  - Sets up test data
+  - Runs CLI tests
+  - Runs pytest
+
+### 2. **Comprehensive CI/CD** (`.github/workflows/ci.yml`)
+- **Purpose**: Full CI/CD pipeline
+- **Features**:
+  - Linting (Ruff, Black)
+  - Unit tests (Python 3.10, 3.11, 3.12)
+  - Integration tests
+  - Security scanning (Safety, Bandit)
+  - Coverage reporting
+  - Package building (on main branch)
+
+### 3. **Multi-OS Testing** (`.github/workflows/multi-os-test.yml`)
+- **Purpose**: Cross-platform testing
+- **OS**: Ubuntu, macOS (Windows not included currently but may be reintroduced)
+- **Python**: 3.10, 3.11, 3.12
+- **Features**: Tests compatibility across different operating systems
+
+### 4. **Basic Test Suite** (`.github/workflows/test.yml`)
+- **Purpose**: Original test workflow
+- **Features**: 
+  - Multiple Python versions
+  - System dependency installation
+  - Test data creation
+  - Coverage reporting
+
+## Setup Scripts
+
+### Test Data Setup (`.github/scripts/setup_test_data.py`)
+Creates dummy test files when example data is not available:
+- PDF documents
+- CSV files
+- Word documents
+- Images
+- Allow/deny lists
+- OCR output files
+
+## Usage
+
+### Running Tests Locally
+
+```bash
+# Install dependencies
+pip install -r requirements.txt
+pip install pytest pytest-cov
+
+# Setup test data
+python .github/scripts/setup_test_data.py
+
+# Run tests
+cd test
+python cli_epilog_suite.py
+```
+
+### GitHub Actions Triggers
+
+1. **Push to main/dev**: Runs all tests
+2. **Pull Request**: Runs tests and linting
+3. **Daily Schedule**: Runs tests at 2 AM UTC
+4. **Manual Trigger**: Can be triggered manually from GitHub
+
+## Configuration
+
+### Environment Variables
+- `PYTHON_VERSION`: Default Python version (3.11)
+- `PYTHONPATH`: Set automatically for test discovery
+
+### Caching
+- Pip dependencies are cached for faster builds
+- Cache key based on requirements.txt hash
+
+### Artifacts
+- Test results (JUnit XML)
+- Coverage reports (HTML, XML)
+- Security reports
+- Build artifacts (on main branch)
+
+## Test Data
+
+The workflows automatically create test data when example files are missing:
+
+### Required Files Created:
+- `example_data/example_of_emails_sent_to_a_professor_before_applying.pdf`
+- `example_data/combined_case_notes.csv`
+- `example_data/Bold minimalist professional cover letter.docx`
+- `example_data/example_complaint_letter.jpg`
+- `example_data/test_allow_list_*.csv`
+- `example_data/partnership_toolkit_redact_*.csv`
+- `example_data/example_outputs/doubled_output_joined.pdf_ocr_output.csv`
+
+### Dependencies Installed:
+- **System**: tesseract-ocr, poppler-utils, OpenGL libraries
+- **Python**: All requirements.txt packages + pytest, reportlab, pillow
+
+## Workflow Status
+
+### Success Criteria:
+- ✅ All tests pass
+- ✅ No linting errors
+- ✅ Security checks pass
+- ✅ Coverage meets threshold (if configured)
+
+### Failure Handling:
+- Tests are designed to skip gracefully if files are missing
+- AWS tests are expected to fail without credentials
+- System dependency failures are handled with fallbacks
+
+## Customization
+
+### Adding New Tests:
+1. Add test methods to `test/cli_epilog_suite.py` or pytest files under `test/test_*.py`
+2. Update test data in `setup_test_data.py` if needed
+3. Tests will automatically run in all workflows
+
+### Modifying Workflows:
+1. Edit the appropriate `.yml` file
+2. Test locally first
+3. Push to trigger the workflow
+
+### Environment-Specific Settings:
+- **Ubuntu**: Full system dependencies
+- **Windows**: Python packages only
+- **macOS**: Homebrew dependencies
+
+## Troubleshooting
+
+### Common Issues:
+
+1. **Missing Dependencies**:
+   - Check system dependency installation
+   - Verify Python package versions
+
+2. **Test Failures**:
+   - Check test data creation
+   - Verify file paths
+   - Review test output logs
+
+3. **AWS Test Failures**:
+   - Expected without credentials
+   - Tests are designed to handle this gracefully
+
+4. **System Dependency Issues**:
+   - Different OS have different requirements
+   - Check the specific OS section in workflows
+
+### Debug Mode:
+Add `--verbose` or `-v` flags to pytest commands for more detailed output.
+
+## Security
+
+- Dependencies are scanned with Safety
+- Code is scanned with Bandit
+- No secrets are exposed in logs
+- Test data is temporary and cleaned up
+
+## Performance
+
+- Tests run in parallel where possible
+- Dependencies are cached
+- Only necessary system packages are installed
+- Test data is created efficiently
+
+## Monitoring
+
+- Workflow status is visible in GitHub Actions tab
+- Coverage reports are uploaded to Codecov
+- Test results are available as artifacts
+- Security reports are generated and stored

.github/workflows/archive_workflows/multi-os-test.yml

@@ -0,0 +1,115 @@
+name: Multi-OS Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    env:
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest] # windows-latest, not included as tesseract cannot be installed silently
+        python-version: ["3.11", "3.12", "3.13"]
+        exclude:
+          # Exclude some combinations to reduce CI time
+          #- os: windows-latest
+          #  python-version: ["3.12", "3.13"]
+          - os: macos-latest
+            python-version: ["3.12", "3.13"]
+
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Install system dependencies (Ubuntu)
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Install system dependencies (macOS)
+      if: matrix.os == 'macos-latest'
+      run: |
+        brew install tesseract poppler
+    
+    - name: Install system dependencies (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        # Create tools directory
+        if (!(Test-Path "C:\tools")) {
+            mkdir C:\tools
+        }
+        
+        # Download and install Tesseract
+        $tesseractUrl = "https://github.com/tesseract-ocr/tesseract/releases/download/5.5.0/tesseract-ocr-w64-setup-5.5.0.20241111.exe"
+        $tesseractInstaller = "C:\tools\tesseract-installer.exe"
+        Invoke-WebRequest -Uri $tesseractUrl -OutFile $tesseractInstaller
+        
+        # Install Tesseract silently
+        Start-Process -FilePath $tesseractInstaller -ArgumentList "/S", "/D=C:\tools\tesseract" -Wait
+        
+        # Download and extract Poppler
+        $popplerUrl = "https://github.com/oschwartz10612/poppler-windows/releases/download/v25.07.0-0/Release-25.07.0-0.zip"
+        $popplerZip = "C:\tools\poppler.zip"
+        Invoke-WebRequest -Uri $popplerUrl -OutFile $popplerZip
+        
+        # Extract Poppler
+        Expand-Archive -Path $popplerZip -DestinationPath C:\tools\poppler -Force
+        
+        # Add to PATH
+        echo "C:\tools\tesseract" >> $env:GITHUB_PATH
+        echo "C:\tools\poppler\poppler-25.07.0\Library\bin" >> $env:GITHUB_PATH
+        
+        # Set environment variables for your application
+        echo "TESSERACT_FOLDER=C:\tools\tesseract" >> $env:GITHUB_ENV
+        echo "POPPLER_FOLDER=C:\tools\poppler\poppler-25.07.0\Library\bin" >> $env:GITHUB_ENV
+        echo "TESSERACT_DATA_FOLDER=C:\tools\tesseract\tessdata" >> $env:GITHUB_ENV
+        
+        # Verify installation using full paths (since PATH won't be updated in current session)
+        & "C:\tools\tesseract\tesseract.exe" --version
+        & "C:\tools\poppler\poppler-25.07.0\Library\bin\pdftoppm.exe" -v
+    
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pytest pytest-cov reportlab pillow
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+    
+    - name: Run CLI tests
+      run: |
+        cd test
+        python cli_epilog_suite.py
+    
+    - name: Run tests with pytest
+      run: |
+        pytest test/ -v --tb=short

.github/workflows/ci.yml

@@ -0,0 +1,269 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+  #schedule:
+  # Run tests daily at 2 AM UTC
+  #  - cron: '0 2 * * *'
+
+permissions:
+  contents: read
+  actions: read
+  pull-requests: write
+  issues: write
+
+env:
+  PYTHON_VERSION: "3.11"
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install ruff black
+    
+    - name: Run Ruff linter
+      run: ruff check .
+    
+    - name: Run Black formatter check
+      run: black --check .
+
+  test-unit:
+    runs-on: ubuntu-latest
+    env:
+      # Avoid optional VLM/torch import path in tools.run_vlm (not installed in lightweight CI deps)
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    strategy:
+      matrix:
+        python-version: [3.11, 3.12, 3.13]
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Cache pip dependencies
+      uses: actions/cache@v5
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements_lightweight.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_lightweight.txt
+        pip install pytest pytest-cov pytest-html pytest-xdist reportlab pillow
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+        echo "Setup script completed. Checking results:"
+        ls -la doc_redaction/example_data/ || echo "doc_redaction/example_data directory not found"
+    
+    - name: Verify test data files
+      run: |
+        echo "Checking if critical test files exist:"
+        ls -la doc_redaction/example_data/
+        echo "Checking for specific PDF files:"
+        ls -la doc_redaction/example_data/*.pdf || echo "No PDF files found"
+        echo "Checking file sizes:"
+        find doc_redaction/example_data -name "*.pdf" -exec ls -lh {} \;
+    
+    - name: Clean up problematic config files
+      run: |
+        rm -f config*.py || true
+    
+    - name: Run CLI tests
+      run: |
+        cd test
+        python cli_epilog_suite.py
+    
+    - name: Run tests with pytest (JUnit and coverage)
+      run: |
+        pytest test/ -v --tb=short \
+          --junitxml=test-results.xml \
+          --cov=. --cov-config=.coveragerc \
+          --cov-report=xml --cov-report=html --cov-report=term
+    
+    #- name: Upload coverage to Codecov - not necessary
+    #  uses: codecov/codecov-action@v3
+    #  if: matrix.python-version == '3.11'
+    #  with:
+    #    file: ./coverage.xml
+    #    flags: unittests
+    #    name: codecov-umbrella
+    #    fail_ci_if_error: false
+    
+    - name: Upload test results
+      uses: actions/upload-artifact@v6
+      if: always()
+      with:
+        name: test-results-python-${{ matrix.python-version }}
+        path: |
+          test-results.xml
+          htmlcov/
+          coverage.xml
+
+  test-integration:
+    runs-on: ubuntu-latest
+    needs: [lint, test-unit]
+    env:
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_lightweight.txt
+        pip install pytest pytest-cov reportlab pillow
+    
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+        echo "Setup script completed. Checking results:"
+        ls -la doc_redaction/example_data/ || echo "doc_redaction/example_data directory not found"
+    
+    - name: Verify test data files
+      run: |
+        echo "Checking if critical test files exist:"
+        ls -la doc_redaction/example_data/
+        echo "Checking for specific PDF files:"
+        ls -la doc_redaction/example_data/*.pdf || echo "No PDF files found"
+        echo "Checking file sizes:"
+        find doc_redaction/example_data -name "*.pdf" -exec ls -lh {} \;
+    
+    - name: Run integration tests
+      run: |
+        cd test
+        python demo_single_test.py
+    
+    - name: Test CLI help
+      run: |
+        python cli_redact.py --help
+    
+    - name: Test CLI version
+      run: |
+        python -c "import sys; print(f'Python {sys.version}')"
+
+  security:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install safety bandit
+    
+    #- name: Run safety scan - removed as now requires login
+    #  run: |
+    #    safety scan -r requirements.txt
+    
+    - name: Run bandit security check
+      run: |
+        bandit -r . -f json -o bandit-report.json || true
+    
+    - name: Upload security report
+      uses: actions/upload-artifact@v6
+      if: always()
+      with:
+        name: security-report
+        path: bandit-report.json
+
+  build:
+    runs-on: ubuntu-latest
+    needs: [lint, test-unit]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+    
+    - name: Build package
+      run: |
+        python -m build
+    
+    - name: Check package
+      run: |
+        twine check dist/*
+    
+    - name: Upload build artifacts
+      uses: actions/upload-artifact@v6
+      with:
+        name: dist
+        path: dist/

.github/workflows/simple-test.yml

@@ -0,0 +1,74 @@
+name: Simple Test Run
+
+on:
+  push:
+    branches: [ dev ]
+  pull_request:
+    branches: [ dev ]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.12"
+    
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_lightweight.txt
+        pip install pytest pytest-cov reportlab pillow
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+        echo "Setup script completed. Checking results:"
+        ls -la doc_redaction/example_data/ || echo "doc_redaction/example_data directory not found"
+    
+    - name: Verify test data files
+      run: |
+        echo "Checking if critical test files exist:"
+        ls -la doc_redaction/example_data/
+        echo "Checking for specific PDF files:"
+        ls -la doc_redaction/example_data/*.pdf || echo "No PDF files found"
+        echo "Checking file sizes:"
+        find doc_redaction/example_data -name "*.pdf" -exec ls -lh {} \;
+    
+    - name: Run CLI tests
+      run: |
+        cd test
+        python cli_epilog_suite.py
+    
+    - name: Run tests with pytest
+      run: |
+        pytest test/ -v --tb=short

.github/workflows/sync_to_hf.yml

@@ -0,0 +1,54 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 1      # Only get the latest state
+          lfs: true           # Download actual LFS files so they can be pushed
+
+      - name: Install Git LFS
+        run: git lfs install
+
+      - name: Recreate repo history (single-commit force push)
+        run: |
+          # 1. Capture the message BEFORE we delete the .git folder
+          COMMIT_MSG=$(git log -1 --pretty=%B)
+          echo "Syncing commit message: $COMMIT_MSG"
+
+          # 2. DELETE the .git folder. 
+          # This turns the repo into a standard folder of files.
+          rm -rf .git
+
+          # 3. Re-initialize a brand new git repo
+          git init -b main
+          git config --global user.name "$HF_USERNAME"
+          git config --global user.email "$HF_EMAIL"
+          
+          # 4. Re-install LFS (needs to be done after git init)
+          git lfs install
+          
+          # 5. Add the remote
+          git remote add hf https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$HF_REPO_ID
+          
+          # 6. Add all files
+          # Since this is a fresh init, Git sees EVERY file as "New"
+          git add .
+          
+          # 7. Commit and Force Push
+          git commit -m "Sync: $COMMIT_MSG"
+          git push --force hf main
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+          HF_EMAIL: ${{ secrets.HF_EMAIL }}
+          HF_REPO_ID: ${{ secrets.HF_REPO_ID }}

.github/workflows/sync_to_hf_zero_gpu.yml

@@ -0,0 +1,59 @@
+name: Sync to Hugging Face hub Zero GPU
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  sync-to-hub-zero-gpu:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 1      # Only get the latest state
+          lfs: true           # Download actual LFS files so they can be pushed
+
+      - name: Install Git LFS
+        run: git lfs install
+
+      # HF Spaces read Space config from README.md front matter. The repo README
+      # targets GitHub (e.g. docker); patch only this CI checkout before HF push.
+      - name: Apply HF Zero GPU Space README front matter
+        run: python3 tools/apply_hf_zero_gpu_readme_frontmatter.py
+
+      - name: Recreate repo history (single-commit force push)
+        run: |
+          # 1. Capture the message BEFORE we delete the .git folder
+          COMMIT_MSG=$(git log -1 --pretty=%B)
+          echo "Syncing commit message: $COMMIT_MSG"
+
+          # 2. DELETE the .git folder. 
+          # This turns the repo into a standard folder of files.
+          rm -rf .git
+
+          # 3. Re-initialize a brand new git repo
+          git init -b main
+          git config --global user.name "$HF_USERNAME"
+          git config --global user.email "$HF_EMAIL"
+          
+          # 4. Re-install LFS (needs to be done after git init)
+          git lfs install
+          
+          # 5. Add the remote
+          git remote add hf https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$HF_REPO_ID_ZERO_GPU
+          
+          # 6. Add all files
+          # Since this is a fresh init, Git sees EVERY file as "New"
+          git add .
+          
+          # 7. Commit and Force Push
+          git commit -m "Sync: $COMMIT_MSG"
+          git push --force hf main
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+          HF_EMAIL: ${{ secrets.HF_EMAIL }}
+          HF_REPO_ID_ZERO_GPU: ${{ secrets.HF_REPO_ID_ZERO_GPU }}

.gitignore

@@ -0,0 +1,59 @@
+*.url
+*.ipynb
+*.pyc
+*.qmd
+_quarto.yml
+quarto_site/*
+src/*
+redaction_deps/*
+.venv/*
+examples/*
+processing/*
+input/*
+output/*
+tools/__pycache__/*
+old_code/*
+tesseract/*
+poppler/*
+build/*
+dist/*
+build_deps/*
+logs/*
+usage/*
+feedback/*
+config/*
+workspace/*
+user_guide/*
+_extensions/*
+doc_redaction.egg-info/*
+.venv_pypi_test/*
+cdk/config/*
+cdk/cdk.out/*
+cdk/archive/*
+tld/*
+tmp/*
+docs/*
+.pi/*
+cdk.out/*
+cdk.json
+cdk.context.json
+precheck.context.json
+.quarto/*
+/.quarto/
+/_site/
+test/config/*
+test/feedback/*
+test/input/*
+test/logs/*
+test/output/*
+test/tmp/*
+test/usage/*
+.ruff_cache/*
+model_cache/*
+sanitized_file/*
+src/doc_redaction.egg-info/*
+docker_compose/*
+**/*.quarto_ipynb
+skills/example_prompts/*
+.pi/sessions/
+docker/pi/agent/sessions/

AGENTS.md

@@ -0,0 +1,113 @@
+# AGENTS.md
+
+Context for AI coding agents working on **doc_redaction** (PII redaction for PDFs, images, Word, and tabular files). Human-oriented docs: [README.md](README.md). User guide: [doc_redaction user guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html).
+
+## Project overview
+
+- **Stack**: Python 3.10+, Gradio UI ([app.py](app.py)), optional FastAPI when `RUN_FASTAPI` is enabled, AWS/LLM integrations via [tools/config.py](tools/config.py) and env files under `config/`.
+- **License**: AGPL-3.0-only (see [pyproject.toml](pyproject.toml)). Respect license terms when adding dependencies.
+- **Accuracy**: Outputs are not guaranteed complete; downstream use should assume **human review** of redacted material.
+
+## Cursor skills: redaction workflow (optional)
+
+For agents operating the deployed app (Gradio Client, review CSV, `/review_apply`), these repo-local playbooks are a suggested ladder:
+
+0. **[`skills/doc-redaction-task-prompt/TASK_PROMPT_TEMPLATE.md`](skills/doc-redaction-task-prompt/TASK_PROMPT_TEMPLATE.md)** — copy-paste user task prompt (Pass 1 default, Pass 2 gated); **user redaction requirements go at the end of the prompt**.
+1. **[`skills/doc-redaction-app/SKILL.md`](skills/doc-redaction-app/SKILL.md)** — first-pass redaction (`/doc_redact` / `/redact_document`) and downloading artifacts.
+2. **[`skills/doc-redact-page-review/SKILL.md`](skills/doc-redact-page-review/SKILL.md)** — after outputs exist: **parallel per-page** child agents, merge into one full-document `*_review_file.csv`, **single** `/review_apply` from the parent.
+3. **[`skills/doc-redaction-modifications/SKILL.md`](skills/doc-redaction-modifications/SKILL.md)** — CSV mechanics, `preview_redaction_boxes`, `/review_apply` patterns, verification, VLM and PyMuPDF fallbacks (single-thread edits and the **technical** reference for page-review children).
+
+## Setup
+
+1. **System**: Install **Tesseract** and **Poppler** (required for OCR/PDF). See [README.md](README.md) (Windows/Linux sections).
+2. **Python**: Create a venv, then install the project (e.g. `pip install -e ".[dev]"` or follow README).
+3. **Configuration**: Copy or edit environment/config as described in README / `config/` (e.g. `app_config.env`). Do not commit secrets.
+
+## Run locally
+
+- Gradio/FastAPI entrypoint is [app.py](app.py). With FastAPI enabled, typical pattern is `uvicorn app:app --host 0.0.0.0 --port 7860` (exact host/port from your config).
+- OpenAPI docs: `/docs` when the FastAPI app is mounted.
+
+## Tests
+
+- Run from repo root: `pytest` (optional: `pytest test/`).
+- Fix failures related to your changes before opening a PR.
+
+## Line order (local OCR and simple text extraction)
+
+Multi-column layouts use shared logic in [`tools/ocr_reading_order.py`](tools/ocr_reading_order.py). Controlled by **`LOCAL_OCR_READING_ORDER`** (`column` default, `legacy` for previous top-left behaviour).
+
+### Local OCR (Paddle/Tesseract)
+
+Word boxes are merged into line-level CSV rows in [`combine_ocr_results`](tools/custom_image_analyser_engine.py).
+
+- **`column`**: detect text columns, assign line numbers down each column left-to-right; full-width lines (headers) first. Stops cross-column merging that produced wide erroneous lines on multi-column PDFs. **Auto-fallback**: the page is treated as single-column unless a *consecutive cluster* of gutter rows (y-gap between adjacent rows ≤ `OCR_COLUMN_MAX_CONSECUTIVE_GUTTER_GAP`, default `0.06` of page height) has ≥ `OCR_COLUMN_MIN_GUTTER_ROWS` (default `3`) rows **and** the cluster's topmost row is above the footer zone (`OCR_COLUMN_FOOTER_ZONE_FRACTION`, default `0.75`). This prevents isolated header bands (logo | title, 1 gutter row), signature-only blocks at the page bottom (cluster starts at y ≥ 0.75), or the combination of both, from forcing column mode on the single-column body text between them.
+- **`PADDLE_PRESERVE_LINE_BOXES=True`** or **`CONVERT_LINE_TO_WORD_LEVEL=False`** with Paddle: keep Paddle line boxes (skip word split + regrouping); line numbers still use column reading order.
+
+### Simple text extraction (PyMuPDF)
+
+[`redact_text_pdf`](tools/file_redaction.py) → [`process_page_to_structured_ocr_pymupdf`](tools/file_redaction.py) calls [`reorder_structured_text_lines`](tools/ocr_reading_order.py) after collecting lines, using **`page.mediabox`** width/height for full-span header detection.
+
+`reorder_structured_text_lines` now mirrors `build_line_groups` (local OCR route):
+
+1. **Column-aware sort** (`sort_reading_order` / `assign_layout_boxes` / `detect_column_split_xpoints`) — or legacy top-left for single-column pages.
+2. **Y-band grouping** (`group_into_lines`) — merges any same-row PyMuPDF lines that were emitted as separate objects (e.g. mixed-font spans) and splits horizontally-disparate boxes via `_finalize_line`.  *Column mode only.*
+3. **Secondary sub-column pass** (`_reorder_lines_column_major`) — ensures correct column-major order when sub-columns sit within a single macro-column.  *Column mode only.*
+4. When a group contains more than one box, constituent boxes are **merged** into a single `OCRResult` (union bbox, joined text, concatenated chars/words).
+
+In single-column / legacy mode only step 1 is applied; PyMuPDF lines are pre-formed so no merging is needed.
+
+### Tunables (both routes)
+
+`OCR_FULL_SPAN_WIDTH_RATIO`, `OCR_COLUMN_GAP_MIN_FRACTION`, `OCR_COLUMN_GUTTER_MIN_FRACTION`, `OCR_COLUMN_SUBGUTTER_MIN_FRACTION` (default `0.015` — fine-grained gutter scan in `assign_layout_boxes`; lower = detects narrower sub-column boundaries), `OCR_COLUMN_MIN_GUTTER_ROWS`, `OCR_COLUMN_MAX_BOX_HEIGHT_RATIO`, `OCR_COLUMN_MAX_CONSECUTIVE_GUTTER_GAP`, `OCR_COLUMN_FOOTER_ZONE_FRACTION`, `OCR_LINE_SPLIT_GAP_FRACTION` (default 0.025 — horizontal gap fraction that forces a line split; must be below the narrowest column gutter, ~0.030 for two-page spreads; also used as the gap threshold for the secondary sub-column sort in `build_line_groups`), `OCR_LINE_Y_THRESHOLD_FRACTION` (default 0.013 — row-alignment tolerance as a fraction of page height; reduced from 0.015 to correctly separate tightly-set 10 pt body text whose row spacing is ~0.014), `OCR_LINE_Y_THRESHOLD_MIN_PX`.
+
+**Sub-column ordering** (`build_line_groups`): after the primary word-level column sort, a second pass (`_reorder_lines_column_major`) clusters the produced line groups by their leftmost x-position using `OCR_LINE_SPLIT_GAP_FRACTION` as the gap threshold. This ensures that adjacent narrow sub-columns whose word-level centre gap is below `column_gap_threshold` (e.g. two columns on a spread where each page is already one macro-column) are still output in left-to-right column-major order rather than interleaved by y-position.
+
+**Fine-grained gutter-based column assignment** (`assign_layout_boxes`): before falling back to centre-gap clustering, `detect_column_split_xpoints` scans the page for structural gutters at the finer `OCR_COLUMN_SUBGUTTER_MIN_FRACTION` threshold (default 0.015). Each qualifying gutter cluster produces a `(split_x, y_min)` pair — the split point is only applied to boxes whose `top ≥ y_min`, preventing a narrow sub-column gutter (visible only in the lower two-column section) from mis-splitting a full-width introductory paragraph that sits above it. This correctly separates narrow adjacent columns (e.g. 1.9 % gutter on a two-page spread) without fragmenting full-width headings or paragraphs.
+
+Changing line order affects PII page text, duplicate-page detection, and review CSV line indices on multi-column documents; re-review after upgrading.
+
+## Agentic / programmatic access (two surfaces)
+
+### 1. FastAPI Agent API (recommended for LLM agents: small JSON bodies)
+
+When `RUN_FASTAPI` is true, routes are mounted under **`/agent`** ([agent_routes.py](agent_routes.py)).
+
+- **Catalog**: `GET /agent/operations` — maps each Gradio `api_name` to an HTTP path and notes whether the route is implemented via CLI or returns HTTP 501 for Gradio-only flows.
+- **Implemented POST routes** (CLI- or [tools/simplified_api.py](tools/simplified_api.py)-backed where noted):  
+  `redact_document`, `redact_data`, `find_duplicate_pages`, `find_duplicate_tabular`, `summarise_document`, `combine_review_pdfs`, `combine_review_csvs`, `export_review_redaction_overlay`, `export_review_page_ocr_visualisation`, `apply_review_redactions`, **`verify_redaction_coverage`** (Pass 1 QA: `must_redact` / `must_not_redact` regex lists, optional `redacted_pdf_path`, optional `auto_prune_suspicious` + `pruned_output_path`; returns `pass_strict`, `pass_with_cleanup`, `pages_flagged_for_vlm`, `pages_needing_csv_cleanup`), **`word_level_ocr_text_search`** (headless word OCR search with optional review-box overlap flags).
+
+**Optional post-redaction Pass 1 QA (main app / CLI):** When `POST_REDACT_PASS1_QA=True` in [`tools/config.py`](tools/config.py) (or `config/app_config.env`), initial redaction emits `*_coverage_report.json` beside the review CSV and optionally `*_review_file_pruned.csv` (sibling, when `POST_REDACT_PASS1_AUTO_PRUNE=True`). Uses deny/allow lists and/or `POST_REDACT_PASS1_MUST_REDACT_PATH` / `POST_REDACT_PASS1_MUST_NOT_REDACT_PATH`. CLI overrides: `--post-redact-pass1-qa`, `--post-redact-pass1-auto-prune`. This is pre-review-apply sanity QA only — agent Pass 1 (policy edits + `/review_apply`) remains separate.  
+  Note: on Gradio ([app.py](app.py)), the Review-tab visual exports use `api_name` **`page_redaction_review_image`** and **`page_ocr_review_image`**; the **`/agent`** routes above keep the explicit `export_review_*` names for the same operations.
+- **Gradio-only stubs** (501 + JSON hint): `load_and_prepare_documents_or_data`.
+- **Auth**: If `AGENT_API_KEY` is set in the environment, send header `X-Agent-API-Key` with that value.
+- **Paths**: Inputs must resolve to files under the repo root, `INPUT_FOLDER`, or `OUTPUT_FOLDER` (see router validation).
+
+Implementation uses **`cli_redact.main(direct_mode_args=...)`** where a CLI task exists (same behaviour as [cli_redact.py](cli_redact.py)); `apply_review_redactions` calls [tools/simplified_api.py](tools/simplified_api.py) instead.
+
+### 2. Gradio Client API (e.g. Hugging Face Spaces)
+
+For remote Spaces or any Gradio deployment exposing the HTTP API:
+
+- **Schema**: `GET https://<host>/gradio_api/info`
+- **Call**: `POST https://<host>/gradio_api/call/{api_name}` with body `{"data":[...]}` (argument order matches the named endpoint’s component list).
+- **Poll**: `GET https://<host>/gradio_api/call/{api_name}/{event_id}`
+- **Hugging Face**: `Authorization: Bearer $HF_TOKEN`
+
+Named `api_name` values in this app include: `redact_document`, `load_and_prepare_documents_or_data`, `apply_review_redactions`, **`doc_redact`** (simple `gr.api`: one PDF/image + optional OCR/PII knobs; returns `(output_paths, message)`; `api_name='/doc_redact'`; parameters include `document_file`, `redact_entities`, `output_dir`, `ocr_method`, `pii_method`, `allow_list`, `deny_list`, `page_min`, `page_max`, **`handwrite_signature_checkbox`** — AWS Textract extraction options such as `Extract handwriting` / `Extract signatures`), **`review_apply`** (simple `gr.api`: PDF + `*_review_file.csv`; returns `(output_paths, message)`; `api_name='/review_apply'`), **`preview_boxes`** (simple `gr.api`: PDF + `*_review_file.csv`; renders proposed boxes onto the original PDF and returns `(zip_path, message)` — use to verify coordinates *before* calling `review_apply`, no redaction applied; `api_name='/preview_boxes'`), **`pdf_summarise`** (simple `gr.api`: PDF + optional summarisation/OCR knobs; returns `(output_paths, status_message, summary_text)`; `api_name='/pdf_summarise'`), **`tabular_redact`** (simple `gr.api`: one tabular file (CSV/XLSX/Parquet/DOCX) + optional knobs; returns `(output_paths, message)`; `api_name='/tabular_redact'`), **`page_redaction_review_image`** (short review overlay export; `api_name='/page_redaction_review_image'`), **`page_ocr_review_image`** (short OCR visualisation export; `api_name='/page_ocr_review_image'`), `word_level_ocr_text_search`, `redact_data`, `find_duplicate_pages`, `find_duplicate_tabular`, `summarise_document`, `combine_review_csvs`, `combine_review_pdfs`. The matching **`POST /agent`** names for those two visual exports are `export_review_redaction_overlay` and `export_review_page_ocr_visualisation` (§1). Many endpoints require **many positional arguments** (full Gradio state); prefer the short `gr.api` routes above or **`POST /agent/apply_review_redactions`** where applicable instead of building the full `data` array from `/gradio_api/info`.
+
+## CLI parity
+
+For scripting and tests, `python cli_redact.py` with flags is authoritative; programmatic merges use `get_cli_default_args_dict()` in [cli_redact.py](cli_redact.py).
+
+## Security and data handling
+
+- Do not commit API keys, tokens, or customer data.
+- Treat paths as untrusted outside validated roots (see [tools/secure_path_utils.py](tools/secure_path_utils.py)).
+- Optional `instruction` / LLM fields must not be passed into shell or unconstrained config keys.
+
+## Conventions for PRs
+
+- Keep changes focused; avoid drive-by refactors.
+- Match existing naming and patterns in [app.py](app.py) and [tools/](tools/).
+- Update tests when behaviour changes; run `pytest` before merge.

Dockerfile

@@ -0,0 +1,232 @@
+# Stage 1: Build dependencies and download models
+FROM public.ecr.aws/docker/library/python:3.12.13-slim-trixie AS builder
+
+# Install system dependencies
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y --no-install-recommends \
+        g++ \
+        make \
+        cmake \
+        unzip \
+        libcurl4-openssl-dev \
+        git \
+    && pip install --upgrade pip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /src
+
+COPY requirements_lightweight.txt .
+
+RUN pip install --verbose --no-cache-dir --target=/install -r requirements_lightweight.txt && rm requirements_lightweight.txt
+
+ARG INSTALL_GRADIO_MCP=False
+ENV INSTALL_GRADIO_MCP=${INSTALL_GRADIO_MCP}
+
+RUN if [ "$INSTALL_GRADIO_MCP" = "True" ]; then \
+    pip install --verbose --no-cache-dir --force-reinstall --target=/install "gradio[mcp]<=6.10.0"; \
+fi
+
+# Optionally install PaddleOCR if the INSTALL_PADDLEOCR environment variable is set to True. Note that GPU-enabled PaddleOCR is unlikely to work in the same environment as a GPU-enabled version of PyTorch, so it is recommended to install PaddleOCR as a CPU-only version if you want to use GPU-enabled PyTorch.
+
+ARG INSTALL_PADDLEOCR=False
+ENV INSTALL_PADDLEOCR=${INSTALL_PADDLEOCR}
+
+ARG PADDLE_GPU_ENABLED=False
+ENV PADDLE_GPU_ENABLED=${PADDLE_GPU_ENABLED}
+
+RUN if [ "$INSTALL_PADDLEOCR" = "True" ] && [ "$PADDLE_GPU_ENABLED" = "False" ]; then \
+    pip install --verbose --no-cache-dir --target=/install "protobuf<=7.34.0" && \
+    pip install --verbose --no-cache-dir --target=/install "paddlepaddle<=3.2.1" && \
+    pip install --verbose --no-cache-dir --target=/install "paddleocr<=3.3.0"; \
+elif [ "$INSTALL_PADDLEOCR" = "True" ] && [ "$PADDLE_GPU_ENABLED" = "True" ]; then \
+    pip install --verbose --no-cache-dir --target=/install "protobuf<=7.34.0" && \
+    pip install --verbose --no-cache-dir --target=/install "paddlepaddle-gpu<=3.2.1" --index-url https://www.paddlepaddle.org.cn/packages/stable/cu129/ && \
+    pip install --verbose --no-cache-dir --target=/install "paddleocr<=3.3.0"; \
+fi
+
+ARG INSTALL_VLM=False
+ENV INSTALL_VLM=${INSTALL_VLM}
+
+ARG TORCH_GPU_ENABLED=False
+ENV TORCH_GPU_ENABLED=${TORCH_GPU_ENABLED}
+
+# Optionally install VLM/LLM packages if the INSTALL_VLM environment variable is set to True.
+RUN if [ "$INSTALL_VLM" = "True" ] && [ "$TORCH_GPU_ENABLED" = "False" ]; then \
+    pip install --verbose --no-cache-dir --target=/install \
+    "torch==2.9.1+cpu" \
+    "torchvision==0.24.1+cpu" \
+    "transformers<=5.5.4" \
+    "accelerate<=1.13.0" \
+    "bitsandbytes<=0.49.2" \
+    "sentencepiece<=0.2.1" \
+    --extra-index-url https://download.pytorch.org/whl/cpu; \
+elif [ "$INSTALL_VLM" = "True" ] && [ "$TORCH_GPU_ENABLED" = "True" ]; then \
+    pip install --verbose --no-cache-dir --target=/install "torch<=2.8.0" --index-url https://download.pytorch.org/whl/cu129 && \
+    pip install --verbose --no-cache-dir --target=/install "torchvision<=0.23.0" --index-url https://download.pytorch.org/whl/cu129 && \
+    pip install --verbose --no-cache-dir --target=/install \
+        "transformers<=5.5.4" \
+        "accelerate<=1.13.0" \
+        "bitsandbytes<=0.49.2" \
+        "sentencepiece<=0.2.1" && \
+    pip install --verbose --no-cache-dir --target=/install "optimum<=2.1.0" && \
+    pip install --verbose --no-cache-dir --target=/install  https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.8cxx11abiTRUE-cp312-cp312-linux_x86_64.whl && \
+    pip install --verbose --no-cache-dir --target=/install  https://github.com/ModelCloud/GPTQModel/releases/download/v5.8.0/gptqmodel-5.8.0+cu128torch2.8-cp312-cp312-linux_x86_64.whl; \
+fi
+
+# ===================================================================
+# Stage 2: A common base for both Lambda and Gradio
+# ===================================================================
+FROM public.ecr.aws/docker/library/python:3.12.13-slim-trixie AS base
+
+# MUST re-declare ARGs in every stage where they are used in RUN commands
+ARG TORCH_GPU_ENABLED=False
+ARG PADDLE_GPU_ENABLED=False
+
+ENV TORCH_GPU_ENABLED=${TORCH_GPU_ENABLED}
+ENV PADDLE_GPU_ENABLED=${PADDLE_GPU_ENABLED}
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    tesseract-ocr \
+    poppler-utils \
+    libgl1 \
+    libglib2.0-0 && \
+    if [ "$TORCH_GPU_ENABLED" = "True" ] || [ "$PADDLE_GPU_ENABLED" = "True" ]; then \
+        apt-get install -y --no-install-recommends libgomp1; \
+    fi && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV APP_HOME=/home/user
+
+# Set env variables for Gradio & other apps
+ENV GRADIO_TEMP_DIR=/tmp/gradio_tmp/ \
+    MPLCONFIGDIR=/tmp/matplotlib_cache/ \
+    GRADIO_OUTPUT_FOLDER=$APP_HOME/app/output/ \
+    GRADIO_INPUT_FOLDER=$APP_HOME/app/input/ \
+    FEEDBACK_LOGS_FOLDER=$APP_HOME/app/feedback/ \
+    ACCESS_LOGS_FOLDER=$APP_HOME/app/logs/ \
+    USAGE_LOGS_FOLDER=$APP_HOME/app/usage/ \
+    CONFIG_FOLDER=$APP_HOME/app/config/ \
+    XDG_CACHE_HOME=/tmp/xdg_cache/user_1000 \
+    TESSERACT_DATA_FOLDER=/usr/share/tessdata \
+    GRADIO_SERVER_NAME=0.0.0.0 \
+    GRADIO_SERVER_PORT=7860 \
+    PATH=$APP_HOME/.local/bin:$PATH \
+    PYTHONPATH=$APP_HOME/app \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    GRADIO_ALLOW_FLAGGING=never \
+    GRADIO_NUM_PORTS=1 \    
+    GRADIO_ANALYTICS_ENABLED=False
+
+# Copy Python packages from the builder stage
+COPY --from=builder /install /usr/local/lib/python3.12/site-packages/
+COPY --from=builder /install/bin /usr/local/bin/
+
+# Reinstall protobuf into the final site-packages. Builder uses multiple `pip install --target=/install`
+# passes; that can break the `google` namespace so `google.protobuf` is missing and Paddle fails at import.
+RUN pip install --no-cache-dir "protobuf<=7.34.0"
+
+# English pipeline is not a normal PyPI dependency; bundle it in the image so runtime works offline.
+# Placed before COPY app code so application changes do not invalidate this layer.
+RUN python -m spacy download en_core_web_lg
+
+# Copy your application code and entrypoint
+COPY . ${APP_HOME}/app
+COPY entrypoint.sh ${APP_HOME}/app/entrypoint.sh
+# Fix line endings and set execute permissions
+RUN sed -i 's/\r$//' ${APP_HOME}/app/entrypoint.sh \
+    && chmod +x ${APP_HOME}/app/entrypoint.sh
+
+WORKDIR ${APP_HOME}/app
+
+# ===================================================================
+# FINAL Stage 3: The Lambda Image (runs as root for simplicity)
+# ===================================================================
+FROM base AS lambda
+# Set runtime ENV for Lambda mode
+ENV APP_MODE=lambda
+ENTRYPOINT ["/home/user/app/entrypoint.sh"]
+CMD ["lambda_entrypoint.lambda_handler"]
+
+# ===================================================================
+# FINAL Stage 4: The Gradio Image (runs as a secure, non-root user)
+# ===================================================================
+FROM base AS gradio
+# Set runtime ENV for Gradio mode
+ENV APP_MODE=gradio
+
+# Create non-root user
+RUN useradd -m -u 1000 user
+
+# Create the base application directory and set its ownership
+RUN mkdir -p ${APP_HOME}/app && chown user:user ${APP_HOME}/app
+
+# Create required sub-folders within the app directory and set their permissions
+# This ensures these specific directories are owned by 'user'
+RUN mkdir -p \
+    ${APP_HOME}/app/output \
+    ${APP_HOME}/app/input \
+    ${APP_HOME}/app/logs \
+    ${APP_HOME}/app/usage \
+    ${APP_HOME}/app/feedback \
+    ${APP_HOME}/app/config \
+    && chown user:user \
+    ${APP_HOME}/app/output \
+    ${APP_HOME}/app/input \
+    ${APP_HOME}/app/logs \
+    ${APP_HOME}/app/usage \
+    ${APP_HOME}/app/feedback \
+    ${APP_HOME}/app/config \
+    && chmod 755 \
+    ${APP_HOME}/app/output \
+    ${APP_HOME}/app/input \
+    ${APP_HOME}/app/logs \
+    ${APP_HOME}/app/usage \
+    ${APP_HOME}/app/feedback \
+    ${APP_HOME}/app/config 
+
+# Now handle the /tmp and /var/tmp directories and their subdirectories, paddle, spacy, tessdata
+RUN mkdir -p /tmp/gradio_tmp /tmp/tld /tmp/matplotlib_cache /tmp /var/tmp ${XDG_CACHE_HOME} \
+    && chown user:user /tmp /var/tmp /tmp/gradio_tmp /tmp/tld /tmp/matplotlib_cache ${XDG_CACHE_HOME} \
+    && chmod 1777 /tmp /var/tmp /tmp/gradio_tmp /tmp/tld /tmp/matplotlib_cache \
+    && chmod 700 ${XDG_CACHE_HOME} \
+    && mkdir -p ${APP_HOME}/.paddlex \
+    && chown user:user ${APP_HOME}/.paddlex \
+    && chmod 755 ${APP_HOME}/.paddlex \
+    && mkdir -p ${APP_HOME}/.local/share/spacy/data \
+    && chown user:user ${APP_HOME}/.local/share/spacy/data \
+    && chmod 755 ${APP_HOME}/.local/share/spacy/data \
+    && mkdir -p /usr/share/tessdata \
+    && chown user:user /usr/share/tessdata \
+    && chmod 755 /usr/share/tessdata
+
+# Fix apply user ownership to all files in the home directory
+RUN chown -R user:user /home/user
+
+# Set permissions for Python executable
+RUN chmod 755 /usr/local/bin/python
+
+# Declare volumes (NOTE: runtime mounts will override permissions — handle with care)
+VOLUME ["/tmp/matplotlib_cache"]
+VOLUME ["/tmp/gradio_tmp"]
+VOLUME ["/tmp/tld"]
+VOLUME ["/home/user/app/output"]
+VOLUME ["/home/user/app/input"]
+VOLUME ["/home/user/app/logs"]
+VOLUME ["/home/user/app/usage"]
+VOLUME ["/home/user/app/feedback"]
+VOLUME ["/home/user/app/config"]
+VOLUME ["/home/user/.paddlex"]
+VOLUME ["/home/user/.local/share/spacy/data"]
+VOLUME ["/usr/share/tessdata"]
+VOLUME ["/tmp"]
+VOLUME ["/var/tmp"]
+
+USER user
+
+EXPOSE $GRADIO_SERVER_PORT
+
+ENTRYPOINT ["/home/user/app/entrypoint.sh"]
+CMD ["python", "app.py"]

Dockerfile.pi

@@ -0,0 +1,40 @@
+# syntax=docker/dockerfile:1
+
+FROM node:22-bookworm-slim
+
+ENV NODE_ENV=production
+ENV DEBIAN_FRONTEND=noninteractive
+ENV NPM_CONFIG_LOGLEVEL=warn
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONPATH=/workspace/doc_redaction
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash \
+    git \
+    curl \
+    ca-certificates \
+    procps \
+    python3 \
+    python3-pip \
+    python3-venv \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN npm install -g --ignore-scripts @earendil-works/pi-coding-agent
+
+COPY requirements_pi_agent.txt /tmp/requirements_pi_agent.txt
+RUN pip3 install --no-cache-dir --break-system-packages \
+    -r /tmp/requirements_pi_agent.txt \
+    && rm /tmp/requirements_pi_agent.txt
+
+RUN mkdir -p /home/node/.pi/agent/sessions /workspace/doc_redaction \
+    && chown -R node:node /home/node/.pi /workspace
+
+WORKDIR /workspace/doc_redaction
+
+USER node
+
+RUN pi --version
+
+ENTRYPOINT ["pi"]
+CMD []

MANIFEST.in

@@ -0,0 +1,4 @@
+recursive-include doc_redaction/assets *.png
+recursive-include doc_redaction/example_data *
+recursive-include intros *.txt
+

README.md

@@ -0,0 +1,346 @@
+---
+title: Document redaction
+emoji: 📝
+colorFrom: blue
+colorTo: yellow
+sdk: docker
+app_file: app.py
+pinned: true
+license: agpl-3.0
+short_description: OCR / redact PDF documents and tabular data
+---
+# Document redaction (doc_redaction)
+
+<a href="https://pypi.org/project/doc-redaction/" target="_blank"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/doc-redaction"></a> 
+
+Redact personally identifiable information (PII) from documents (PDF, PNG, JPG), Word files (DOCX), or tabular data (XLSX/CSV/Parquet). Please see the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for a full walkthrough of all the features in the app.
+
+---
+
+## 🚀 Quick Start - Installation and first run
+
+Follow these instructions to get the document redaction application running on your local machine.
+
+### 1. Package installation
+
+#### Option 1 - Recommended: Install from source repo
+
+Clone the repository and install in editable mode:
+
+```bash
+git clone https://github.com/seanpedrick-case/doc_redaction.git
+cd doc_redaction
+pip install -e .
+```
+
+##### Install extras (Paddle or Transformers/Torch VLM)
+
+To install with PaddleOCR:
+
+```bash
+pip install -e ".[paddle]"
+```
+
+Note that the versions of both PaddleOCR and Torch installed by default are the CPU-only versions. If you want to install the equivalent GPU versions, you will need to run the following commands:
+```bash
+pip install paddlepaddle-gpu==3.2.1 --index-url https://www.paddlepaddle.org.cn/packages/stable/cu129/
+```
+
+If you want to run VLMs / LLMs with the transformers package:
+
+```bash
+pip install -e ".[vlm]"
+```
+
+
+**Note:** It is difficult to get paddlepaddle gpu working in an environment alongside torch. You may well need to reinstall the cpu version to ensure compatibility, and run paddlepaddle-gpu in a separate environment without torch installed. If you get errors related to .dll files following paddle gpu install, you may need to install the latest c++ redistributables. For Windows, you can find them [here](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170)
+
+```bash
+pip install torch==2.8.0 --index-url https://download.pytorch.org/whl/cu129
+pip install torchvision --index-url https://download.pytorch.org/whl/cu129
+```
+
+#### Option 2 - Install from PyPI
+
+Create a virtual environment (recommended) and install **doc_redaction**.
+
+```bash
+python -m venv venv
+# Windows:
+.\venv\Scripts\activate
+# macOS/Linux:
+source venv/bin/activate
+```
+
+The package is published on PyPI as **`doc-redaction`** (import name **`doc_redaction`**):
+
+```bash
+pip install doc_redaction
+```
+
+Optional extras (same as in `pyproject.toml`). For installing paddleOCR:
+
+```bash
+pip install "doc_redaction[paddle]"
+```
+
+For running VLMs / LLMs with the transformers package:
+
+```bash
+pip install "doc_redaction[vlm]"
+```
+
+For programmatic use (CLI-first API matching Gradio `api_name` routes), see **[Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html)**. The console script **`cli_redact`** is available after install.
+
+**Web UI from a PyPI install:** You *can* start the Gradio UI after `pip install doc_redaction` by running (note that the prerequisites tesseract and poppler will need to be correctly installed following step 2 below):
+
+```bash
+python -m app
+```
+
+**Important: your working directory matters.** When you run `python -m app`, the app treats your *current folder* as the “app folder”:
+
+- It will look for configuration at `config/app_config.env` *relative to the folder you run it from* (and `python -m doc_redaction.install_deps` will also write `config/app_config.env` there).
+- It may create new folders in that location (for example `config/`, `output/`, `input/`, `logs/`, `usage/`, `feedback/`, and temporary/cache folders depending on your settings).
+- The UI example files and bundled assets are packaged with the PyPI install (they live inside the installed `doc_redaction` package). If you run from a “random” directory after a PyPI install, the app can still locate its packaged examples; your working directory mainly affects where `config/`, `input/`, `output/`, logs, and temp folders are created.
+
+In practice, the **smoothest UI experience** (examples, bundled assets, docs links, predictable relative paths) is still usually via a **repository checkout** or **Docker**, but PyPI install is sufficient to launch the UI as long as you run it from a suitable working folder and have the system dependencies available (or run `python -m doc_redaction.install_deps` first).
+
+#### Option 3 - Docker installation
+
+The doc_redaction Redaction app can be installed by using the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) or Docker compose files ([llama.cpp](https://github.com/ggml-org/llama.cpp), [vLLM](https://docs.vllm.ai/en/stable/)) provided in the repo.
+
+##### With Llama.cpp / vLLM inference server
+
+The project now has Docker and Docker compose files available to pair running the Redaction app with local inference servers powered by [llama.cpp](https://github.com/ggml-org/llama.cpp), or [vLLM](https://docs.vllm.ai/en/stable/). Llama.cpp is more flexible than vLLM for low VRAM systems, as Llama.cpp will offload to cpu/system RAM automatically rather than failing as vLLM tends to do.
+
+For Llama.cpp, you can use the [docker-compose_llama.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_llama.yml) file, and for vLLM, you can use the [docker-compose_vllm.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_vllm.yml) file. To run, Docker / Docker Desktop should be installed, and then you can run the commands suggested in the top of the files to run the servers.
+
+You will need ~40 GB of disk space to run everything depending on the model chosen from the compose file. For the vLLM server, you will need 24 GB VRAM. For the Llama.cpp server, 24 GB VRAM is needed to run at full speed, but the n-gpu-layers and n-cpu-moe parameters in the Docker compose file can be adjusted to fit into your system. I would suggest that 8 GB VRAM is needed as a bare minimum for decent inference speed. See the [Unsloth guide](https://unsloth.ai/docs/models/qwen3.5) for more details on working with GGUF files for Qwen 3.5.
+
+##### Without Llama.cpp / vLLM inference server
+
+If you want a working Docker installation without GPU support, you can install from the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) in the repo. A working example of this, with the CPU version of PaddleOCR, can be found on [Hugging Face](https://huggingface.co/spaces/seanpedrickcase/document_redaction). You can adjust the INSTALL_PADDLEOCR, PADDLE_GPU_ENABLED, INSTALL_VLM, and TORCH_GPU_ENABLED config variables to adjust for PaddleOCR and Transformers packages for local VLM support. Note that GPU-enabled PaddleOCR, and GPU-enabled Transformers/Torch often don't work well together, which is one reason why a Llama.cpp/vLLM inference server Docker installation option is provided below.
+
+### 2. Install prerequisites: Tesseract and Poppler
+
+This application relies on two external tools for OCR (Tesseract) and PDF processing (Poppler). Please install them on your system before proceeding. To run the Document Redaction app successfully, these tools need to be installed and either 1. added to PATH, or 2. be in a folder that is directly referenced in the config/app_config.env file with the variables TESSERACT_FOLDER and POPPLER_FOLDER (defined [here](https://github.com/seanpedrick-case/doc_redaction/blob/main/tools/config.py) if you want to see the code). The instructions below will guide you through diffferent ways to install these dependencies.
+
+---
+
+#### Automated dependency setup (recommended)
+
+If you **don’t have admin rights** (or you just want the simplest setup), you can have the project download and configure **Tesseract** and **Poppler** into a local `redaction_deps/` folder inside the doc_redaction folder.
+
+You need the installer script available first, which means either:
+
+- **Repository checkout**: `git clone ...` and run the command from the repo root (recommended for the web UI), or
+- **PyPI install**: `pip install doc_redaction` and run from a writable folder where you want `redaction_deps/` and `config/app_config.env` to be created/updated.
+
+From the repository root (or your chosen working folder) after creating/activating your venv and installing Python requirements:
+
+```bash
+python -m doc_redaction.install_deps
+```
+
+This writes `TESSERACT_FOLDER` / `POPPLER_FOLDER` into `config/app_config.env` so the app can find the binaries without you editing your system PATH.
+
+To just check whether your machine can already see the tools:
+
+```bash
+python -m doc_redaction.install_deps --verify-only
+```
+
+#### **On Windows**
+
+If you don’t use the automated setup above, you can install the dependencies manually by downloading installers and adding the programs to your system's PATH.
+
+1.  **Install Tesseract OCR:**
+    *   Download the installer from the official Tesseract at [UB Mannheim page](https://github.com/UB-Mannheim/tesseract/wiki) (e.g., `tesseract-ocr-w64-setup-v5.X.X...exe`).
+    *   Run the installer.
+    *   **IMPORTANT:** During installation, ensure you select the option to "Add Tesseract to system PATH for all users" or a similar option. This is crucial for the application to find the Tesseract executable.
+
+
+2.  **Install Poppler:**
+    *   Download the latest Poppler binary for Windows. A common source is the [Poppler for Windows](https://github.com/oschwartz10612/poppler-windows) GitHub releases page. Download the `.zip` file (e.g., `poppler-25.07.0-win.zip`).
+    *   Extract the contents of the zip file to a permanent location on your computer, for example, `C:\Program Files\poppler\`.
+    *   You must add the `bin` folder from your Poppler installation to your system's PATH environment variable.
+        *   Search for "Edit the system environment variables" in the Windows Start Menu and open it.
+        *   Click the "Environment Variables..." button.
+        *   In the "System variables" section, find and select the `Path` variable, then click "Edit...".
+        *   Click "New" and add the full path to the `bin` directory inside your Poppler folder (e.g., `C:\Program Files\poppler\poppler-24.02.0\bin`).
+        *   Click OK on all windows to save the changes.
+
+    To verify, open a new Command Prompt and run `tesseract --version` and `pdftoppm -v`. If they both return version information, you have successfully installed the prerequisites.
+---
+
+#### **On Linux (Debian/Ubuntu)**
+
+Open your terminal and run the following command to install Tesseract and Poppler:
+
+```bash
+sudo apt-get update && sudo apt-get install -y tesseract-ocr poppler-utils
+```
+
+#### **On Linux (Fedora/CentOS/RHEL)**
+
+Open your terminal and use the `dnf` or `yum` package manager:
+
+```bash
+sudo dnf install -y tesseract poppler-utils
+```
+---
+
+### 3. Run the Application
+
+With all dependencies installed, you can now start the Gradio application GUI. For a guide on how to use this, please go [here](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html).
+
+```bash
+python app.py
+```
+
+After running the command, the application will start, and you will see a local URL in your terminal (usually `http://127.0.0.1:7860`).
+
+Open this URL in your web browser to use the document redaction tool
+
+#### Command line interface
+
+For example CLI commands, please refer to [this guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html#command-line-interface-cli) or the examples in [cli_redact.py](https://github.com/seanpedrick-case/doc_redaction/blob/main/cli_redact.py#L321)
+
+If you installed from **PyPI**, use the installed console script:
+
+```bash
+cli_redact --help
+```
+
+From a **repository checkout**, you can also run:
+
+```bash
+python cli_redact.py --help
+```
+
+#### Python package commands
+
+For Python examples in using the Python package, please see [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+
+---
+
+
+### 4. ⚙️ Configuration (Optional)
+
+You can customise the application's behavior by creating a configuration file. This allows you to change settings without modifying the source code, such as enabling AWS features, changing logging behavior, or pointing to local Tesseract/Poppler installations. A full overview of all the potential settings you can modify in the app_config.env file can be seen in tools/config.py, with explanation on the documentation website for [the github repo](https://seanpedrick-case.github.io/doc_redaction/)
+
+To get started:
+1.  Locate the `example_config.env` file in the root of the project.
+2.  Create a new file named `app_config.env` inside the `config/` directory (i.e., `config/app_config.env`).
+3.  Copy the contents from `example_config.env` into your new `config/app_config.env` file.
+4.  Modify the values in `config/app_config.env` to suit your needs. The application will automatically load these settings on startup.
+
+If you do not create this file, the application will run with default settings.
+
+#### Configuration Breakdown
+
+Here is an overview of the most important settings, separated by whether they are for local use or require AWS.
+
+---
+
+#### **Local & General Settings (No AWS Required)**
+
+These settings are useful for all users, regardless of whether you are using AWS.
+
+*   `TESSERACT_FOLDER` / `POPPLER_FOLDER`
+    *   Use these if you installed Tesseract or Poppler to a custom location on **Windows** and did not add them to the system PATH.
+    *   Provide the path to the respective installation folders (for Poppler, point to the `bin` sub-directory).
+    *   **Examples:** `POPPLER_FOLDER=C:/Program Files/poppler-24.02.0/bin/` `TESSERACT_FOLDER=tesseract/`
+
+*   `TESSERACT_DATA_FOLDER`
+    *   If Tesseract runs but you see an error like `Error opening data file ./eng.traineddata` or `Tesseract couldn't load any languages`, this is usually because it can't find the `tessdata/` language files.
+    *   Set this to the folder that contains `eng.traineddata` (typically a `tessdata` directory).
+    *   **Examples (Windows):** `TESSERACT_DATA_FOLDER=C:/Program Files/Tesseract-OCR/tessdata`
+
+*   `SHOW_LANGUAGE_SELECTION=True`
+    *   Set to `True` to display a language selection dropdown in the UI for OCR processing.
+
+*   `DEFAULT_LOCAL_OCR_MODEL=tesseract`"
+    *   Choose the backend for local OCR. Options are `tesseract`, `paddle`, or `hybrid`. "Tesseract" is the default, and is recommended. "hybrid-paddle" is a combination of the two - first pass through the redactions will be done with Tesseract, and then a second pass will be done with PaddleOCR on words with low confidence. "paddle" will only return whole line text extraction, and so will only work for OCR, not redaction. 
+
+*   `SESSION_OUTPUT_FOLDER=False`
+    *   If `True`, redacted files will be saved in unique subfolders within the `output/` directory for each session.
+
+*   `DISPLAY_FILE_NAMES_IN_LOGS=False`
+    *   For privacy, file names are not recorded in usage logs by default. Set to `True` to include them.
+
+---
+
+#### **AWS-Specific Settings**
+
+These settings are only relevant if you intend to use AWS services like Textract for OCR and Comprehend for PII detection.
+
+*   `RUN_AWS_FUNCTIONS=True`
+    *   **This is the master switch.** You must set this to `True` to enable any AWS functionality. If it is `False`, all other AWS settings will be ignored.
+
+*   **UI Options:**
+    *   `SHOW_AWS_TEXT_EXTRACTION_OPTIONS=True`: Adds "AWS Textract" as an option in the text extraction dropdown.
+    *   `SHOW_AWS_PII_DETECTION_OPTIONS=True`: Adds "AWS Comprehend" as an option in the PII detection dropdown.
+
+*   **Core AWS Configuration:**
+    *   `AWS_REGION=example-region`: Set your AWS region (e.g., `us-east-1`).
+    *   `DOCUMENT_REDACTION_BUCKET=example-bucket`: The name of the S3 bucket the application will use for temporary file storage and processing.
+
+*   **AWS Logging:**
+    *   `SAVE_LOGS_TO_DYNAMODB=True`: If enabled, usage and feedback logs will be saved to DynamoDB tables.
+    *   `ACCESS_LOG_DYNAMODB_TABLE_NAME`, `USAGE_LOG_DYNAMODB_TABLE_NAME`, etc.: Specify the names of your DynamoDB tables for logging.
+
+*   **Advanced AWS Textract Features:**
+    *   `SHOW_WHOLE_DOCUMENT_TEXTRACT_CALL_OPTIONS=True`: Enables UI components for large-scale, asynchronous document processing via Textract.
+    *   `TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET=example-bucket-output`: A separate S3 bucket for the final output of asynchronous Textract jobs.
+    *   `LOAD_PREVIOUS_TEXTRACT_JOBS_S3=True`: If enabled, the app will try to load the status of previously submitted asynchronous jobs from S3.
+
+*   **Cost Tracking (for internal accounting):**
+    *   `SHOW_COSTS=True`: Displays an estimated cost for AWS operations. Can be enabled even if AWS functions are off.
+    *   `GET_COST_CODES=True`: Enables a dropdown for users to select a cost code before running a job.
+    *   `COST_CODES_PATH=config/cost_codes.csv`: The local path to a CSV file containing your cost codes.
+    *   `ENFORCE_COST_CODES=True`: Makes selecting a cost code mandatory before starting a redaction.
+
+Now you have the app installed, please refer to the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for more information on how to use it for basic and advanced redaction.
+
+## For agents (API quickstart)
+
+If you are an LLM/agent interacting with this app over HTTP (e.g. Hugging Face Spaces), **do not guess inputs** from the UI. Use the Gradio schema as the source of truth:
+
+- **Discover schema**: `GET /gradio_api/info`
+- **Upload files**: `POST /gradio_api/upload` (multipart field `files`) → returns server-internal paths like `/tmp/gradio_tmp/...`
+- **Call**: `POST /gradio_api/call/{api_name}` with body `{"data":[...]}` (argument order must match `/gradio_api/info`)
+- **Poll**: `GET /gradio_api/call/{api_name}/{event_id}` until complete
+- **Download outputs**: `GET /gradio_api/file={path}` (note: some deployments return 403 without session cookies)
+
+### Choose the correct route (prefer short `gr.api` endpoints)
+
+Fetch `/gradio_api/info` and then prefer the simplest route that exists:
+
+- **Apply edited review CSV to a PDF**: `/review_apply`
+- **Redact a PDF/image document**: `/doc_redact` — optional `handwrite_signature_checkbox` for AWS Textract (e.g. `Extract handwriting`, `Extract signatures`)
+- **Summarise a PDF**: `/pdf_summarise`
+- **Redact tabular files (CSV/XLSX/Parquet/DOCX)**: `/tabular_redact`
+
+If those endpoints are not present in your deployment, fall back to the long UI-chained routes (`/apply_review_redactions`, `/redact_data`, etc.) and build `data[]` strictly from `/gradio_api/info`.
+
+### Common gotchas
+
+- **Arity errors** (`needed: N, got: M`) mean you called a session-heavy UI handler with the wrong `data[]`. Prefer the short endpoints above.
+- **`handle_file()` gotcha** (for `gradio_client` users): do **not** wrap server-internal upload paths (e.g. `/tmp/gradio_tmp/...`) with `handle_file()`. Pass them as plain strings.
+- **Container-only outputs**: outputs may be written to container paths (e.g. `/home/user/app/output/`). Plan to download via `file=...` or use a mounted output directory in Docker.
+
+### Optional: MCP server
+
+If you want external agents to call this app reliably without re-implementing Gradio upload/call/poll/download details, consider an **MCP server** that wraps the main tasks (`redact_document`, `apply_review_redactions`, `redact_tabular`, `summarise_document`) behind a small tool interface. See the [relevant documentation](https://github.com/seanpedrick-case/doc_redaction/blob/main/mcp_doc_redaction/README.md).
+
+**Use as a library:** After installing from [PyPI](https://pypi.org/project/doc-redaction/) (`pip install doc_redaction`), you can call the same workflows as the Gradio `api_name` routes from Python. See the documentation: [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+    
+To extract text from documents, the 'Local' options are PikePDF for PDFs with selectable text, and OCR with Tesseract. Use AWS Textract to extract more complex elements e.g. handwriting, signatures, or unclear text. PaddleOCR and VLM support is also provided (see the installation instructions below). 
+
+For PII identification, 'Local' (based on spaCy) gives good results if you are looking for common names or terms, or a custom list of terms to redact (see Redaction settings).  AWS Comprehend gives better results at a small cost.
+
+Additional options on the 'Redaction settings' include, the type of information to redact (e.g. people, places), custom terms to include/ exclude from redaction, fuzzy matching, language settings, and whole page redaction. After redaction is complete, you can view and modify suggested redactions on the 'Review redactions' tab to quickly create a final redacted document.
+
+NOTE: The app is not 100% accurate, and it will miss some personal information. It is essential that all outputs are reviewed **by a human** before using the final outputs.

README_PYPI.md

@@ -0,0 +1,330 @@
+# Document redaction (doc_redaction)
+
+<a href="https://pypi.org/project/doc-redaction/" target="_blank"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/doc-redaction"></a> 
+
+Redact personally identifiable information (PII) from documents (PDF, PNG, JPG), Word files (DOCX), or tabular data (XLSX/CSV/Parquet). Please see the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for a full walkthrough of all the features in the app.
+
+---
+
+## 🚀 Quick Start - Installation and first run
+
+Follow these instructions to get the document redaction application running on your local machine.
+
+### 1. Package installation
+
+#### Option 1 - Recommended: Install from source repo
+
+Clone the repository and install in editable mode:
+
+```bash
+git clone https://github.com/seanpedrick-case/doc_redaction.git
+cd doc_redaction
+pip install -e .
+```
+
+##### Install extras (Paddle or Transformers/Torch VLM)
+
+To install with PaddleOCR:
+
+```bash
+pip install -e ".[paddle]"
+```
+
+Note that the versions of both PaddleOCR and Torch installed by default are the CPU-only versions. If you want to install the equivalent GPU versions, you will need to run the following commands:
+```bash
+pip install paddlepaddle-gpu==3.2.1 --index-url https://www.paddlepaddle.org.cn/packages/stable/cu129/
+```
+
+If you want to run VLMs / LLMs with the transformers package:
+
+```bash
+pip install -e ".[vlm]"
+```
+
+
+**Note:** It is difficult to get paddlepaddle gpu working in an environment alongside torch. You may well need to reinstall the cpu version to ensure compatibility, and run paddlepaddle-gpu in a separate environment without torch installed. If you get errors related to .dll files following paddle gpu install, you may need to install the latest c++ redistributables. For Windows, you can find them [here](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170)
+
+```bash
+pip install torch==2.8.0 --index-url https://download.pytorch.org/whl/cu129
+pip install torchvision --index-url https://download.pytorch.org/whl/cu129
+```
+
+#### Option 2 - Install from PyPI
+
+Create a virtual environment (recommended) and install **doc_redaction**.
+
+```bash
+python -m venv venv
+# Windows:
+.\venv\Scripts\activate
+# macOS/Linux:
+source venv/bin/activate
+```
+
+The package is published on PyPI as **`doc-redaction`** (import name **`doc_redaction`**):
+
+```bash
+pip install doc_redaction
+```
+
+Optional extras (same as in `pyproject.toml`). For installing paddleOCR:
+
+```bash
+pip install "doc_redaction[paddle]"
+```
+
+For running VLMs / LLMs with the transformers package:
+
+```bash
+pip install "doc_redaction[vlm]"
+```
+
+For programmatic use (CLI-first API matching Gradio `api_name` routes), see **[Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html)**. The console script **`cli_redact`** is available after install.
+
+**Web UI from a PyPI install:** You *can* start the Gradio UI after `pip install doc_redaction` by running (note that the prerequisites tesseract and poppler will need to be correctly installed following step 2 below):
+
+```bash
+python -m app
+```
+
+**Important: your working directory matters.** When you run `python -m app`, the app treats your *current folder* as the “app folder”:
+
+- It will look for configuration at `config/app_config.env` *relative to the folder you run it from* (and `python -m doc_redaction.install_deps` will also write `config/app_config.env` there).
+- It may create new folders in that location (for example `config/`, `output/`, `input/`, `logs/`, `usage/`, `feedback/`, and temporary/cache folders depending on your settings).
+- The UI example files and bundled assets are packaged with the PyPI install (they live inside the installed `doc_redaction` package). If you run from a “random” directory after a PyPI install, the app can still locate its packaged examples; your working directory mainly affects where `config/`, `input/`, `output/`, logs, and temp folders are created.
+
+In practice, the **smoothest UI experience** (examples, bundled assets, docs links, predictable relative paths) is still usually via a **repository checkout** or **Docker**, but PyPI install is sufficient to launch the UI as long as you run it from a suitable working folder and have the system dependencies available (or run `python -m doc_redaction.install_deps` first).
+
+#### Option 3 - Docker installation
+
+The doc_redaction Redaction app can be installed by using the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) or Docker compose files ([llama.cpp](https://github.com/ggml-org/llama.cpp), [vLLM](https://docs.vllm.ai/en/stable/)) provided in the repo.
+
+##### With Llama.cpp / vLLM inference server
+
+The project now has Docker and Docker compose files available to pair running the Redaction app with local inference servers powered by [llama.cpp](https://github.com/ggml-org/llama.cpp), or [vLLM](https://docs.vllm.ai/en/stable/). Llama.cpp is more flexible than vLLM for low VRAM systems, as Llama.cpp will offload to cpu/system RAM automatically rather than failing as vLLM tends to do.
+
+For Llama.cpp, you can use the [docker-compose_llama.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_llama.yml) file, and for vLLM, you can use the [docker-compose_vllm.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_vllm.yml) file. To run, Docker / Docker Desktop should be installed, and then you can run the commands suggested in the top of the files to run the servers.
+
+You will need ~40 GB of disk space to run everything depending on the model chosen from the compose file. For the vLLM server, you will need 24 GB VRAM. For the Llama.cpp server, 24 GB VRAM is needed to run at full speed, but the n-gpu-layers and n-cpu-moe parameters in the Docker compose file can be adjusted to fit into your system. I would suggest that 8 GB VRAM is needed as a bare minimum for decent inference speed. See the [Unsloth guide](https://unsloth.ai/docs/models/qwen3.5) for more details on working with GGUF files for Qwen 3.5.
+
+##### Without Llama.cpp / vLLM inference server
+
+If you want a working Docker installation without GPU support, you can install from the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) in the repo. A working example of this, with the CPU version of PaddleOCR, can be found on [Hugging Face](https://huggingface.co/spaces/seanpedrickcase/document_redaction). You can adjust the INSTALL_PADDLEOCR, PADDLE_GPU_ENABLED, INSTALL_VLM, and TORCH_GPU_ENABLED config variables to adjust for PaddleOCR and Transformers packages for local VLM support. Note that GPU-enabled PaddleOCR, and GPU-enabled Transformers/Torch often don't work well together, which is one reason why a Llama.cpp/vLLM inference server Docker installation option is provided below.
+
+### 2. Install prerequisites: Tesseract and Poppler
+
+This application relies on two external tools for OCR (Tesseract) and PDF processing (Poppler). Please install them on your system before proceeding.
+
+---
+
+#### Automated dependency setup (recommended)
+
+If you **don’t have admin rights** (or you just want the simplest setup), you can have the project download and configure **Tesseract** and **Poppler** into a local `redaction_deps/` folder inside the doc_redaction folder.
+
+You need the installer script available first, which means either:
+
+- **Repository checkout**: `git clone ...` and run the command from the repo root (recommended for the web UI), or
+- **PyPI install**: `pip install doc_redaction` and run from a writable folder where you want `redaction_deps/` and `config/app_config.env` to be created/updated.
+
+From the repository root (or your chosen working folder) after creating/activating your venv and installing Python requirements:
+
+```bash
+python -m doc_redaction.install_deps
+```
+
+This writes `TESSERACT_FOLDER` / `POPPLER_FOLDER` into `config/app_config.env` so the app can find the binaries without you editing your system PATH.
+
+To just check whether your machine can already see the tools:
+
+```bash
+python -m doc_redaction.install_deps --verify-only
+```
+
+#### **On Windows**
+
+If you don’t use the automated setup above, you can install the dependencies manually by downloading installers and adding the programs to your system's PATH.
+
+1.  **Install Tesseract OCR:**
+    *   Download the installer from the official Tesseract at [UB Mannheim page](https://github.com/UB-Mannheim/tesseract/wiki) (e.g., `tesseract-ocr-w64-setup-v5.X.X...exe`).
+    *   Run the installer.
+    *   **IMPORTANT:** During installation, ensure you select the option to "Add Tesseract to system PATH for all users" or a similar option. This is crucial for the application to find the Tesseract executable.
+
+
+2.  **Install Poppler:**
+    *   Download the latest Poppler binary for Windows. A common source is the [Poppler for Windows](https://github.com/oschwartz10612/poppler-windows) GitHub releases page. Download the `.zip` file (e.g., `poppler-25.07.0-win.zip`).
+    *   Extract the contents of the zip file to a permanent location on your computer, for example, `C:\Program Files\poppler\`.
+    *   You must add the `bin` folder from your Poppler installation to your system's PATH environment variable.
+        *   Search for "Edit the system environment variables" in the Windows Start Menu and open it.
+        *   Click the "Environment Variables..." button.
+        *   In the "System variables" section, find and select the `Path` variable, then click "Edit...".
+        *   Click "New" and add the full path to the `bin` directory inside your Poppler folder (e.g., `C:\Program Files\poppler\poppler-24.02.0\bin`).
+        *   Click OK on all windows to save the changes.
+
+    To verify, open a new Command Prompt and run `tesseract --version` and `pdftoppm -v`. If they both return version information, you have successfully installed the prerequisites.
+---
+
+#### **On Linux (Debian/Ubuntu)**
+
+Open your terminal and run the following command to install Tesseract and Poppler:
+
+```bash
+sudo apt-get update && sudo apt-get install -y tesseract-ocr poppler-utils
+```
+
+#### **On Linux (Fedora/CentOS/RHEL)**
+
+Open your terminal and use the `dnf` or `yum` package manager:
+
+```bash
+sudo dnf install -y tesseract poppler-utils
+```
+---
+
+### 3. Run the Application
+
+With all dependencies installed, you can now start the Gradio application GUI. For a guide on how to use this, please go [here](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html).
+
+```bash
+python app.py
+```
+
+After running the command, the application will start, and you will see a local URL in your terminal (usually `http://127.0.0.1:7860`).
+
+Open this URL in your web browser to use the document redaction tool
+
+#### Command line interface
+
+For example CLI commands, please refer to [this guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html#command-line-interface-cli) or the examples in [cli_redact.py](https://github.com/seanpedrick-case/doc_redaction/blob/main/cli_redact.py#L321)
+
+If you installed from **PyPI**, use the installed console script:
+
+```bash
+cli_redact --help
+```
+
+From a **repository checkout**, you can also run:
+
+```bash
+python cli_redact.py --help
+```
+
+#### Python package commands
+
+For Python examples in using the Python package, please see [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+
+---
+
+
+### 4. ⚙️ Configuration (Optional)
+
+You can customise the application's behavior by creating a configuration file. This allows you to change settings without modifying the source code, such as enabling AWS features, changing logging behavior, or pointing to local Tesseract/Poppler installations. A full overview of all the potential settings you can modify in the app_config.env file can be seen in tools/config.py, with explanation on the documentation website for [the github repo](https://seanpedrick-case.github.io/doc_redaction/)
+
+To get started:
+1.  Locate the `example_config.env` file in the root of the project.
+2.  Create a new file named `app_config.env` inside the `config/` directory (i.e., `config/app_config.env`).
+3.  Copy the contents from `example_config.env` into your new `config/app_config.env` file.
+4.  Modify the values in `config/app_config.env` to suit your needs. The application will automatically load these settings on startup.
+
+If you do not create this file, the application will run with default settings.
+
+#### Configuration Breakdown
+
+Here is an overview of the most important settings, separated by whether they are for local use or require AWS.
+
+---
+
+#### **Local & General Settings (No AWS Required)**
+
+These settings are useful for all users, regardless of whether you are using AWS.
+
+*   `TESSERACT_FOLDER` / `POPPLER_FOLDER`
+    *   Use these if you installed Tesseract or Poppler to a custom location on **Windows** and did not add them to the system PATH.
+    *   Provide the path to the respective installation folders (for Poppler, point to the `bin` sub-directory).
+    *   **Examples:** `POPPLER_FOLDER=C:/Program Files/poppler-24.02.0/bin/` `TESSERACT_FOLDER=tesseract/`
+
+*   `SHOW_LANGUAGE_SELECTION=True`
+    *   Set to `True` to display a language selection dropdown in the UI for OCR processing.
+
+*   `DEFAULT_LOCAL_OCR_MODEL=tesseract`"
+    *   Choose the backend for local OCR. Options are `tesseract`, `paddle`, or `hybrid`. "Tesseract" is the default, and is recommended. "hybrid-paddle" is a combination of the two - first pass through the redactions will be done with Tesseract, and then a second pass will be done with PaddleOCR on words with low confidence. "paddle" will only return whole line text extraction, and so will only work for OCR, not redaction. 
+
+*   `SESSION_OUTPUT_FOLDER=False`
+    *   If `True`, redacted files will be saved in unique subfolders within the `output/` directory for each session.
+
+*   `DISPLAY_FILE_NAMES_IN_LOGS=False`
+    *   For privacy, file names are not recorded in usage logs by default. Set to `True` to include them.
+
+---
+
+#### **AWS-Specific Settings**
+
+These settings are only relevant if you intend to use AWS services like Textract for OCR and Comprehend for PII detection.
+
+*   `RUN_AWS_FUNCTIONS=True`
+    *   **This is the master switch.** You must set this to `True` to enable any AWS functionality. If it is `False`, all other AWS settings will be ignored.
+
+*   **UI Options:**
+    *   `SHOW_AWS_TEXT_EXTRACTION_OPTIONS=True`: Adds "AWS Textract" as an option in the text extraction dropdown.
+    *   `SHOW_AWS_PII_DETECTION_OPTIONS=True`: Adds "AWS Comprehend" as an option in the PII detection dropdown.
+
+*   **Core AWS Configuration:**
+    *   `AWS_REGION=example-region`: Set your AWS region (e.g., `us-east-1`).
+    *   `DOCUMENT_REDACTION_BUCKET=example-bucket`: The name of the S3 bucket the application will use for temporary file storage and processing.
+
+*   **AWS Logging:**
+    *   `SAVE_LOGS_TO_DYNAMODB=True`: If enabled, usage and feedback logs will be saved to DynamoDB tables.
+    *   `ACCESS_LOG_DYNAMODB_TABLE_NAME`, `USAGE_LOG_DYNAMODB_TABLE_NAME`, etc.: Specify the names of your DynamoDB tables for logging.
+
+*   **Advanced AWS Textract Features:**
+    *   `SHOW_WHOLE_DOCUMENT_TEXTRACT_CALL_OPTIONS=True`: Enables UI components for large-scale, asynchronous document processing via Textract.
+    *   `TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET=example-bucket-output`: A separate S3 bucket for the final output of asynchronous Textract jobs.
+    *   `LOAD_PREVIOUS_TEXTRACT_JOBS_S3=True`: If enabled, the app will try to load the status of previously submitted asynchronous jobs from S3.
+
+*   **Cost Tracking (for internal accounting):**
+    *   `SHOW_COSTS=True`: Displays an estimated cost for AWS operations. Can be enabled even if AWS functions are off.
+    *   `GET_COST_CODES=True`: Enables a dropdown for users to select a cost code before running a job.
+    *   `COST_CODES_PATH=config/cost_codes.csv`: The local path to a CSV file containing your cost codes.
+    *   `ENFORCE_COST_CODES=True`: Makes selecting a cost code mandatory before starting a redaction.
+
+Now you have the app installed, please refer to the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for more information on how to use it for basic and advanced redaction.
+
+## For agents (API quickstart)
+
+If you are an LLM/agent interacting with this app over HTTP (e.g. Hugging Face Spaces), **do not guess inputs** from the UI. Use the Gradio schema as the source of truth:
+
+- **Discover schema**: `GET /gradio_api/info`
+- **Upload files**: `POST /gradio_api/upload` (multipart field `files`) → returns server-internal paths like `/tmp/gradio_tmp/...`
+- **Call**: `POST /gradio_api/call/{api_name}` with body `{"data":[...]}` (argument order must match `/gradio_api/info`)
+- **Poll**: `GET /gradio_api/call/{api_name}/{event_id}` until complete
+- **Download outputs**: `GET /gradio_api/file={path}` (note: some deployments return 403 without session cookies)
+
+### Choose the correct route (prefer short `gr.api` endpoints)
+
+Fetch `/gradio_api/info` and then prefer the simplest route that exists:
+
+- **Apply edited review CSV to a PDF**: `/review_apply`
+- **Redact a PDF/image document**: `/doc_redact` — optional `handwrite_signature_checkbox` for AWS Textract (e.g. `Extract handwriting`, `Extract signatures`)
+- **Summarise a PDF**: `/pdf_summarise`
+- **Redact tabular files (CSV/XLSX/Parquet/DOCX)**: `/tabular_redact`
+
+If those endpoints are not present in your deployment, fall back to the long UI-chained routes (`/apply_review_redactions`, `/redact_data`, etc.) and build `data[]` strictly from `/gradio_api/info`.
+
+### Common gotchas
+
+- **Arity errors** (`needed: N, got: M`) mean you called a session-heavy UI handler with the wrong `data[]`. Prefer the short endpoints above.
+- **`handle_file()` gotcha** (for `gradio_client` users): do **not** wrap server-internal upload paths (e.g. `/tmp/gradio_tmp/...`) with `handle_file()`. Pass them as plain strings.
+- **Container-only outputs**: outputs may be written to container paths (e.g. `/home/user/app/output/`). Plan to download via `file=...` or use a mounted output directory in Docker.
+
+### Optional: MCP server
+
+If you want external agents to call this app reliably without re-implementing Gradio upload/call/poll/download details, consider an **MCP server** that wraps the main tasks (`redact_document`, `apply_review_redactions`, `redact_tabular`, `summarise_document`) behind a small tool interface. See the [relevant documentation](https://github.com/seanpedrick-case/doc_redaction/blob/main/mcp_doc_redaction/README.md).
+
+**Use as a library:** After installing from [PyPI](https://pypi.org/project/doc-redaction/) (`pip install doc_redaction`), you can call the same workflows as the Gradio `api_name` routes from Python. See the documentation: [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+    
+To extract text from documents, the 'Local' options are PikePDF for PDFs with selectable text, and OCR with Tesseract. Use AWS Textract to extract more complex elements e.g. handwriting, signatures, or unclear text. PaddleOCR and VLM support is also provided (see the installation instructions below). 
+
+For PII identification, 'Local' (based on spaCy) gives good results if you are looking for common names or terms, or a custom list of terms to redact (see Redaction settings).  AWS Comprehend gives better results at a small cost.
+
+Additional options on the 'Redaction settings' include, the type of information to redact (e.g. people, places), custom terms to include/ exclude from redaction, fuzzy matching, language settings, and whole page redaction. After redaction is complete, you can view and modify suggested redactions on the 'Review redactions' tab to quickly create a final redacted document.
+
+NOTE: The app is not 100% accurate, and it will miss some personal information. It is essential that all outputs are reviewed **by a human** before using the final outputs.

agent_routes.py

@@ -0,0 +1,1167 @@
+"""
+FastAPI routes for programmatic / agent callers.
+
+HTTP paths align with Gradio ``api_name`` values in app.py. See GET /agent/operations
+for the full map. Uses cli_redact.main(direct_mode_args=...) where a CLI task exists.
+"""
+
+from __future__ import annotations
+
+import io
+import os
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, Depends, Header, HTTPException
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field, field_validator
+
+from tools.config import (
+    AWS_LLM_PII_OPTION,
+    AWS_PII_OPTION,
+    INFERENCE_SERVER_PII_OPTION,
+    INPUT_FOLDER,
+    LOCAL_OCR_MODEL_OPTIONS,
+    LOCAL_PII_OPTION,
+    LOCAL_TRANSFORMERS_LLM_PII_OPTION,
+    OUTPUT_FOLDER,
+)
+from tools.secure_path_utils import validate_path_safety
+
+router = APIRouter(tags=["Agent"])
+
+REPO_ROOT = Path(__file__).resolve().parent
+_MAX_INSTRUCTION_LEN = 16_000
+
+# NOTE: Paths from request bodies are untrusted. Avoid Path.resolve() on untrusted
+# input (CodeQL py/path-injection); instead normalize via os.path and enforce
+# containment under trusted roots.
+
+# Mirrors app.py api_name values (Gradio).
+GRADIO_API_NAMES: tuple[str, ...] = (
+    "redact_document",
+    "load_and_prepare_documents_or_data",
+    "apply_review_redactions",
+    "review_apply",
+    "pdf_summarise",
+    "tabular_redact",
+    "word_level_ocr_text_search",
+    "redact_data",
+    "find_duplicate_pages",
+    "find_duplicate_tabular",
+    "summarise_document",
+    "combine_review_csvs",
+    "combine_review_pdfs",
+    "export_review_redaction_overlay",
+    "export_review_page_ocr_visualisation",
+    "verify_redaction_coverage",
+)
+
+
+def _allowed_path_roots() -> list[Path]:
+    # Return roots without resolving. These are trusted config values, but avoiding
+    # Path.resolve() keeps CodeQL happy and matches our "no resolve on untrusted"
+    # approach elsewhere.
+    roots = [REPO_ROOT]
+    for folder in (INPUT_FOLDER, OUTPUT_FOLDER):
+        if folder:
+            roots.append(Path(str(folder)))
+    return roots
+
+
+def _sanitize_untrusted_path_input(path_str: str) -> str:
+    """Basic raw-input validation before any path normalization."""
+    if not isinstance(path_str, str):
+        raise HTTPException(status_code=400, detail="Path must be a string.")
+    cleaned = path_str.strip()
+    if not cleaned:
+        raise HTTPException(status_code=400, detail="Path must not be empty.")
+    if "\x00" in cleaned:
+        raise HTTPException(status_code=400, detail="Path contains invalid null byte.")
+    return cleaned
+
+
+def _normalize_untrusted_path_to_abs(path_str: str) -> str:
+    """
+    Expand ~, then normalize to an absolute path.
+
+    Relative paths are interpreted relative to REPO_ROOT (matching prior behaviour).
+    """
+    safe_input = _sanitize_untrusted_path_input(path_str)
+    expanded = os.path.expanduser(safe_input)
+    if os.path.isabs(expanded):
+        return os.path.normpath(os.path.abspath(expanded))
+    return os.path.normpath(os.path.abspath(os.path.join(str(REPO_ROOT), expanded)))
+
+
+def _must_be_under_allowed_roots(candidate_abs: str, original: str) -> None:
+    """Enforce candidate is contained under repo, INPUT_FOLDER, or OUTPUT_FOLDER."""
+    candidate_real = os.path.realpath(str(candidate_abs))
+    allowed_roots = [
+        os.path.realpath(os.path.abspath(str(p))) for p in _allowed_path_roots()
+    ]
+    for root in allowed_roots:
+        try:
+            common = os.path.commonpath([candidate_real, root])
+        except ValueError:
+            # Different drive on Windows or invalid path mix
+            continue
+        if common == root:
+            return
+    raise HTTPException(
+        status_code=403,
+        detail="Path must be under the app repo, INPUT_FOLDER, or OUTPUT_FOLDER",
+    )
+
+
+def _path_must_be_allowed_file(path_str: str) -> str:
+    """Resolve path, ensure it is under an allowed root and exists as a file."""
+    candidate_abs = _normalize_untrusted_path_to_abs(path_str)
+    candidate_real = os.path.realpath(candidate_abs)
+
+    # Validate both "safe path" patterns and containment under trusted roots.
+    _must_be_under_allowed_roots(candidate_real, path_str)
+    ok = any(
+        validate_path_safety(candidate_real, base_path=str(root))
+        for root in _allowed_path_roots()
+    )
+    if not ok:
+        raise HTTPException(status_code=400, detail=f"Unsafe path rejected: {path_str}")
+    try:
+        candidate_path = Path(candidate_real)
+        if not candidate_path.is_file():
+            raise HTTPException(
+                status_code=400, detail=f"Not a file or missing: {candidate_real}"
+            )
+    except OSError:
+        raise HTTPException(
+            status_code=400, detail=f"Not a file or missing: {candidate_real}"
+        )
+    return candidate_real
+
+
+def _path_must_be_allowed_directory(path_str: str, *, must_exist: bool = True) -> str:
+    """
+    Normalize and validate a directory path under allowed roots.
+
+    By default the directory must already exist; callers can opt out (e.g. output_dir
+    that will be created later by the CLI).
+    """
+    candidate_abs = _normalize_untrusted_path_to_abs(path_str)
+    candidate_real = os.path.realpath(candidate_abs)
+
+    _must_be_under_allowed_roots(candidate_real, path_str)
+    ok = any(
+        validate_path_safety(candidate_real, base_path=str(root))
+        for root in _allowed_path_roots()
+    )
+    if not ok:
+        raise HTTPException(status_code=400, detail=f"Unsafe path rejected: {path_str}")
+    if must_exist:
+        try:
+            if not Path(candidate_real).is_dir():
+                raise HTTPException(
+                    status_code=400, detail=f"Not a directory: {candidate_real}"
+                )
+        except OSError:
+            raise HTTPException(
+                status_code=400, detail=f"Not a directory: {candidate_real}"
+            )
+    return candidate_real
+
+
+def _optional_agent_api_key(x_agent_api_key: Optional[str] = Header(None)) -> None:
+    expected = os.environ.get("AGENT_API_KEY", "").strip()
+    if not expected:
+        return
+    if not x_agent_api_key or x_agent_api_key.strip() != expected:
+        raise HTTPException(
+            status_code=401,
+            detail="Set header X-Agent-API-Key to match AGENT_API_KEY environment variable",
+        )
+
+
+class AgentRedactDocumentRequest(BaseModel):
+    """Parity with Gradio api_name ``redact_document``."""
+
+    input_files: list[str] = Field(
+        ...,
+        min_length=1,
+        description="Paths to input files (PDF, images, or tabular/Word for anonymisation)",
+    )
+    instruction: Optional[str] = Field(
+        None,
+        description="Optional instructions for LLM-based PII detection (custom_llm_instructions)",
+    )
+    output_dir: Optional[str] = None
+    input_dir: Optional[str] = None
+    ocr_method: Optional[str] = Field(
+        None,
+        description=(
+            "High-level OCR/text mode. Accepted values: 'Local OCR', "
+            "'AWS Textract', 'Local text'. To choose a specific local OCR engine "
+            "(e.g. paddle/tesseract/vlm), set "
+            "overrides.chosen_local_ocr_model."
+        ),
+    )
+    pii_detector: Optional[str] = Field(
+        None,
+        description=(
+            "PII detection method. Recommended configured labels: "
+            f"'{LOCAL_PII_OPTION}', '{AWS_PII_OPTION}', '{AWS_LLM_PII_OPTION}', "
+            f"'{INFERENCE_SERVER_PII_OPTION}', '{LOCAL_TRANSFORMERS_LLM_PII_OPTION}', "
+            "'None'."
+        ),
+    )
+    overrides: Optional[dict[str, Any]] = Field(
+        None,
+        description=(
+            "Optional CLI flag overrides; keys must match argparse destination names. "
+            "For local OCR model selection, set 'chosen_local_ocr_model' "
+            f"(allowed models depend on deployment; configured options: {LOCAL_OCR_MODEL_OPTIONS})."
+        ),
+    )
+
+    model_config = {
+        "json_schema_extra": {
+            "examples": [
+                {
+                    "input_files": [
+                        "example_data/example_of_emails_sent_to_a_professor_before_applying.pdf"
+                    ],
+                    "instruction": "Do not redact the university name.",
+                    "ocr_method": "Local OCR",
+                    "pii_detector": LOCAL_PII_OPTION,
+                    "overrides": {"chosen_local_ocr_model": "paddle"},
+                }
+            ]
+        }
+    }
+
+    @field_validator("instruction")
+    @classmethod
+    def _cap_instruction(cls, v: Optional[str]) -> Optional[str]:
+        if v is None:
+            return v
+        if len(v) > _MAX_INSTRUCTION_LEN:
+            raise ValueError(f"instruction exceeds {_MAX_INSTRUCTION_LEN} characters")
+        return v
+
+
+class AgentRedactDataRequest(AgentRedactDocumentRequest):
+    """Parity with Gradio api_name ``redact_data``; same CLI task as redact_document."""
+
+
+class AgentTaskResponse(BaseModel):
+    status: str
+    gradio_api_name: str
+    task: str
+    output_dir: str
+    input_dir: str
+    message: str
+    log_excerpt: Optional[str] = None
+    output_paths: Optional[list[str]] = None
+
+
+class AgentVerifyRedactionRequest(BaseModel):
+    review_csv_path: str = Field(..., description="Path to *_review_file.csv")
+    ocr_words_csv_path: str = Field(
+        ..., description="Path to *_ocr_results_with_words_*.csv from the same run"
+    )
+    must_redact: Optional[List[str]] = Field(
+        None,
+        description="Regex patterns for terms that must be covered by review boxes.",
+    )
+    must_not_redact: Optional[List[str]] = Field(
+        None,
+        description="Regex patterns for terms that must not appear in review rows.",
+    )
+    redacted_pdf_path: Optional[str] = Field(
+        None, description="Optional applied *_redacted.pdf for text-layer leak checks."
+    )
+    total_pages: Optional[int] = Field(None, ge=1)
+    min_word_length: int = Field(3, ge=1, le=32)
+    sample_pixels: bool = Field(
+        False,
+        description="Sample pixel darkness at box centres on redacted PDF (requires redacted_pdf_path).",
+    )
+    auto_prune_suspicious: bool = Field(
+        False,
+        description="Remove prunable suspicious short/OCR-fragment rows and write pruned CSV.",
+    )
+    pruned_output_path: Optional[str] = Field(
+        None,
+        description="Output path for pruned CSV when auto_prune_suspicious is true.",
+    )
+
+
+class AgentVerifyRedactionResponse(BaseModel):
+    status: str
+    gradio_api_name: str = "verify_redaction_coverage"
+    coverage_pass: bool
+    coverage_pass_strict: bool
+    coverage_pass_with_cleanup: bool
+    pruned_csv_path: Optional[str] = None
+    prune_log: Optional[Dict[str, Any]] = None
+    report: Dict[str, Any]
+
+
+class AgentWordLevelOcrSearchRequest(BaseModel):
+    ocr_words_csv_path: str = Field(
+        ..., description="Path to *_ocr_results_with_words_*.csv"
+    )
+    search_text: str = Field(..., min_length=3, max_length=500)
+    similarity_threshold: float = Field(1.0, ge=0.0, le=1.0)
+    use_regex: bool = False
+    review_csv_path: Optional[str] = Field(
+        None,
+        description="Optional *_review_file.csv to flag whether each hit is covered by a box.",
+    )
+
+
+class AgentWordLevelOcrSearchResponse(BaseModel):
+    status: str
+    gradio_api_name: str = "word_level_ocr_text_search"
+    result: Dict[str, Any]
+
+
+def _merge_redact_direct_mode(body: AgentRedactDocumentRequest) -> dict[str, Any]:
+    from cli_redact import get_cli_default_args_dict
+
+    merged: dict[str, Any] = get_cli_default_args_dict()
+    merged["task"] = "redact"
+    merged["input_file"] = [_path_must_be_allowed_file(p) for p in body.input_files]
+
+    if body.instruction is not None:
+        merged["custom_llm_instructions"] = body.instruction
+    if body.output_dir is not None:
+        # Output folders may not exist yet (CLI will create). Still constrain to allowed roots.
+        merged["output_dir"] = _path_must_be_allowed_directory(
+            body.output_dir, must_exist=False
+        )
+    if body.input_dir is not None:
+        # Input dir should exist if provided.
+        merged["input_dir"] = _path_must_be_allowed_directory(
+            body.input_dir, must_exist=True
+        )
+    if body.ocr_method is not None:
+        merged["ocr_method"] = body.ocr_method
+    if body.pii_detector is not None:
+        merged["pii_detector"] = body.pii_detector
+
+    if body.overrides:
+        allowed = set(merged.keys())
+        for key, value in body.overrides.items():
+            if key not in allowed:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Unknown override key '{key}'. Must be a known CLI argument name.",
+                )
+            merged[key] = value
+
+    return merged
+
+
+def _run_cli_main(direct: dict[str, Any], gradio_api_name: str) -> AgentTaskResponse:
+    from cli_redact import main as cli_main
+
+    buf = io.StringIO()
+    old_stdout = sys.stdout
+    try:
+        sys.stdout = buf
+        cli_main(direct_mode_args=direct)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e)) from e
+    finally:
+        sys.stdout = old_stdout
+
+    log_excerpt = buf.getvalue()
+    if len(log_excerpt) > 8000:
+        log_excerpt = log_excerpt[-8000:]
+
+    return AgentTaskResponse(
+        status="completed",
+        gradio_api_name=gradio_api_name,
+        task=str(direct.get("task", "")),
+        output_dir=str(direct.get("output_dir", "")),
+        input_dir=str(direct.get("input_dir", "")),
+        message="cli_redact.main finished; see log_excerpt for console output",
+        log_excerpt=log_excerpt or None,
+    )
+
+
+@router.post(
+    "/redact_document",
+    response_model=AgentTaskResponse,
+    summary="redact_document (Gradio api_name)",
+    description=(
+        "Matches Gradio ``api_name='redact_document'``. "
+        "``python cli_redact.py --task redact --input_file ...``. "
+        "Optional ``instruction`` maps to ``custom_llm_instructions``. "
+        "OCR modes: 'Local OCR' | 'AWS Textract' | 'Local text'. "
+        "Specific local OCR engines are set via ``overrides.chosen_local_ocr_model`` "
+        f"(for example: {LOCAL_OCR_MODEL_OPTIONS}). "
+        "PII methods should use configured labels shown on the request schema."
+    ),
+)
+def post_redact_document(
+    body: AgentRedactDocumentRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    direct = _merge_redact_direct_mode(body)
+    return _run_cli_main(direct, "redact_document")
+
+
+@router.post(
+    "/redact_data",
+    response_model=AgentTaskResponse,
+    summary="redact_data (Gradio api_name)",
+    description=(
+        "Matches Gradio ``api_name='redact_data'``. Same CLI ``redact`` task as "
+        "/redact_document; use CSV/XLSX/DOCX paths for tabular/Word flows. "
+        "OCR modes: 'Local OCR' | 'AWS Textract' | 'Local text'. "
+        "Specific local OCR engines are set via ``overrides.chosen_local_ocr_model`` "
+        f"(for example: {LOCAL_OCR_MODEL_OPTIONS}). "
+        "PII methods should use configured labels shown on the request schema."
+    ),
+)
+def post_redact_data(
+    body: AgentRedactDataRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    direct = _merge_redact_direct_mode(body)
+    return _run_cli_main(direct, "redact_data")
+
+
+@router.post(
+    "/tasks/redact",
+    response_model=AgentTaskResponse,
+    summary="Legacy: same as /redact_document",
+    description="Deprecated alias; prefer POST /agent/redact_document.",
+    deprecated=True,
+    include_in_schema=True,
+)
+def post_tasks_redact_legacy(
+    body: AgentRedactDocumentRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    direct = _merge_redact_direct_mode(body)
+    return _run_cli_main(direct, "redact_document")
+
+
+class AgentFindDuplicatePagesRequest(BaseModel):
+    input_files: list[str] = Field(..., min_length=1)
+    similarity_threshold: Optional[float] = None
+    min_word_count: Optional[int] = None
+    min_consecutive_pages: Optional[int] = None
+    greedy_match: Optional[bool] = None
+    combine_pages: Optional[bool] = None
+    overrides: Optional[dict[str, Any]] = None
+
+
+@router.post(
+    "/find_duplicate_pages",
+    response_model=AgentTaskResponse,
+    summary="find_duplicate_pages (Gradio api_name)",
+    description="``cli_redact --task deduplicate --duplicate_type pages``.",
+)
+def post_find_duplicate_pages(
+    body: AgentFindDuplicatePagesRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from cli_redact import get_cli_default_args_dict
+
+    merged = get_cli_default_args_dict()
+    merged["task"] = "deduplicate"
+    merged["duplicate_type"] = "pages"
+    merged["input_file"] = [_path_must_be_allowed_file(p) for p in body.input_files]
+    if body.similarity_threshold is not None:
+        merged["similarity_threshold"] = body.similarity_threshold
+    if body.min_word_count is not None:
+        merged["min_word_count"] = body.min_word_count
+    if body.min_consecutive_pages is not None:
+        merged["min_consecutive_pages"] = body.min_consecutive_pages
+    if body.greedy_match is not None:
+        merged["greedy_match"] = "True" if body.greedy_match else "False"
+    if body.combine_pages is not None:
+        merged["combine_pages"] = "True" if body.combine_pages else "False"
+    if body.overrides:
+        allowed = set(merged.keys())
+        for k, v in body.overrides.items():
+            if k not in allowed:
+                raise HTTPException(400, f"Unknown override key: {k}")
+            merged[k] = v
+    return _run_cli_main(merged, "find_duplicate_pages")
+
+
+class AgentFindDuplicateTabularRequest(BaseModel):
+    input_files: list[str] = Field(..., min_length=1)
+    text_columns: Optional[list[str]] = None
+    similarity_threshold: Optional[float] = None
+    min_word_count: Optional[int] = None
+    overrides: Optional[dict[str, Any]] = None
+
+
+@router.post(
+    "/find_duplicate_tabular",
+    response_model=AgentTaskResponse,
+    summary="find_duplicate_tabular (Gradio api_name)",
+)
+def post_find_duplicate_tabular(
+    body: AgentFindDuplicateTabularRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from cli_redact import get_cli_default_args_dict
+
+    merged = get_cli_default_args_dict()
+    merged["task"] = "deduplicate"
+    merged["duplicate_type"] = "tabular"
+    merged["input_file"] = [_path_must_be_allowed_file(p) for p in body.input_files]
+    if body.text_columns is not None:
+        merged["text_columns"] = body.text_columns
+    if body.similarity_threshold is not None:
+        merged["similarity_threshold"] = body.similarity_threshold
+    if body.min_word_count is not None:
+        merged["min_word_count"] = body.min_word_count
+    if body.overrides:
+        allowed = set(merged.keys())
+        for k, v in body.overrides.items():
+            if k not in allowed:
+                raise HTTPException(400, f"Unknown override key: {k}")
+            merged[k] = v
+    return _run_cli_main(merged, "find_duplicate_tabular")
+
+
+class AgentSummariseDocumentRequest(BaseModel):
+    input_files: list[str] = Field(..., min_length=1)
+    summarisation_inference_method: Optional[str] = None
+    summarisation_format: Optional[str] = None
+    summarisation_context: Optional[str] = None
+    summarisation_additional_instructions: Optional[str] = None
+    overrides: Optional[dict[str, Any]] = None
+
+
+@router.post(
+    "/summarise_document",
+    response_model=AgentTaskResponse,
+    summary="summarise_document (Gradio api_name)",
+)
+def post_summarise_document(
+    body: AgentSummariseDocumentRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from cli_redact import get_cli_default_args_dict
+
+    merged = get_cli_default_args_dict()
+    merged["task"] = "summarise"
+    merged["input_file"] = [_path_must_be_allowed_file(p) for p in body.input_files]
+    if body.summarisation_inference_method is not None:
+        merged["summarisation_inference_method"] = body.summarisation_inference_method
+    if body.summarisation_format is not None:
+        merged["summarisation_format"] = body.summarisation_format
+    if body.summarisation_context is not None:
+        merged["summarisation_context"] = body.summarisation_context
+    if body.summarisation_additional_instructions is not None:
+        merged["summarisation_additional_instructions"] = (
+            body.summarisation_additional_instructions
+        )
+    if body.overrides:
+        allowed = set(merged.keys())
+        for k, v in body.overrides.items():
+            if k not in allowed:
+                raise HTTPException(400, f"Unknown override key: {k}")
+            merged[k] = v
+    return _run_cli_main(merged, "summarise_document")
+
+
+class AgentCombineReviewPdfsRequest(BaseModel):
+    input_files: list[str] = Field(..., min_length=2)
+    output_dir: Optional[str] = None
+
+
+@router.post(
+    "/combine_review_pdfs",
+    response_model=AgentTaskResponse,
+    summary="combine_review_pdfs (Gradio api_name)",
+)
+def post_combine_review_pdfs(
+    body: AgentCombineReviewPdfsRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from cli_redact import get_cli_default_args_dict
+
+    merged = get_cli_default_args_dict()
+    merged["task"] = "combine_review_pdfs"
+    merged["input_file"] = [_path_must_be_allowed_file(p) for p in body.input_files]
+    if body.output_dir is not None:
+        merged["output_dir"] = _path_must_be_allowed_directory(body.output_dir)
+    return _run_cli_main(merged, "combine_review_pdfs")
+
+
+class _NamedPath:
+    """merge_csv_files expects objects with a .name attribute (Gradio file-like)."""
+
+    __slots__ = ("name",)
+
+    def __init__(self, path: str) -> None:
+        self.name = path
+
+
+class AgentCombineReviewCsvsRequest(BaseModel):
+    input_files: list[str] = Field(..., min_length=1)
+    output_dir: Optional[str] = Field(
+        None, description="Defaults to config OUTPUT_FOLDER"
+    )
+
+
+class AgentApplyReviewRedactionsRequest(BaseModel):
+    """Headless parity with Gradio ``api_name='apply_review_redactions'`` (prepare + apply)."""
+
+    pdf_path: str = Field(
+        ...,
+        description="Path to the source PDF under allowed roots.",
+    )
+    review_csv_path: str = Field(
+        ...,
+        description=(
+            "Path to the review plan CSV; basename must contain '_review_file' "
+            "(e.g. mydoc_review_file.csv)."
+        ),
+    )
+    output_dir: Optional[str] = Field(
+        None,
+        description="Output directory (created if missing); defaults to OUTPUT_FOLDER.",
+    )
+    input_dir: Optional[str] = Field(
+        None,
+        description="Input/working directory for page images; defaults to INPUT_FOLDER.",
+    )
+    text_extract_method: Optional[str] = Field(
+        None,
+        description="OCR/text mode passed to prepare (defaults to CLI ocr_method).",
+    )
+    efficient_ocr: Optional[bool] = Field(
+        None,
+        description="If set, overrides EFFICIENT_OCR for the prepare step.",
+    )
+
+
+@router.post(
+    "/combine_review_csvs",
+    response_model=AgentTaskResponse,
+    summary="combine_review_csvs (Gradio api_name)",
+    description="Uses tools.helper_functions.merge_csv_files (not cli_redact).",
+)
+def post_combine_review_csvs(
+    body: AgentCombineReviewCsvsRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from tools.helper_functions import merge_csv_files
+
+    paths = [_NamedPath(_path_must_be_allowed_file(p)) for p in body.input_files]
+    out_dir = body.output_dir or OUTPUT_FOLDER
+    out_dir_resolved = _path_must_be_allowed_directory(str(out_dir), must_exist=True)
+    sep = "/" if not out_dir_resolved.endswith(("/", "\\")) else ""
+    out_files = merge_csv_files(paths, output_folder=out_dir_resolved + sep)
+    return AgentTaskResponse(
+        status="completed",
+        gradio_api_name="combine_review_csvs",
+        task="combine_review_csvs",
+        output_dir=out_dir_resolved,
+        input_dir="",
+        message="merge_csv_files completed",
+        output_paths=out_files,
+    )
+
+
+class AgentExportReviewRedactionOverlayRequest(BaseModel):
+    """Agent JSON body for the same overlay render as Gradio ``api_name='page_redaction_review_image'``."""
+
+    page_image_path: str = Field(
+        ...,
+        description="Path to page raster (PNG/JPEG) used as underlay; must be under allowed roots.",
+    )
+    boxes: List[Dict[str, Any]] = Field(
+        ...,
+        min_length=1,
+        description="Annotator-style boxes: label, color, xmin, ymin, xmax, ymax (normalized 0–1).",
+    )
+    page_number: int = Field(
+        1, ge=1, description="1-based page index for the output filename."
+    )
+    doc_base_name: str = Field(
+        "review",
+        description="Basename for output file (e.g. document name without extension).",
+    )
+    review_df_records: Optional[List[Dict[str, Any]]] = Field(
+        None,
+        description="Optional rows (include at least 'label') for stable label→line-pattern mapping.",
+    )
+    label_abbrev_chars: Optional[int] = Field(
+        None,
+        ge=0,
+        le=24,
+        description="Draw this many leading characters of each label on the image; omit to use REVIEW_OVERLAY_LABEL_ABBREV_CHARS from config (0 = off).",
+    )
+
+
+class AgentExportReviewPageOcrVisualisationRequest(BaseModel):
+    """Agent JSON body for the same OCR visualisation as Gradio ``api_name='page_ocr_review_image'``."""
+
+    page_image_path: str = Field(
+        ...,
+        description="Path to page raster (PNG/JPEG) used as underlay; must be under allowed roots.",
+    )
+    ocr_results: Dict[str, Any] = Field(
+        ...,
+        description="Word-level OCR results dict (line_key -> {words:[{text, bounding_box, conf, ...}]}).",
+    )
+    page_number: int = Field(
+        1, ge=1, description="1-based page index (used for naming)."
+    )
+    doc_base_name: str = Field(
+        "review",
+        description="Basename for output file (e.g. document name without extension).",
+    )
+
+
+@router.post(
+    "/export_review_redaction_overlay",
+    response_model=AgentTaskResponse,
+    summary="export_review_redaction_overlay (Agent API; Gradio api_name: page_redaction_review_image)",
+    description=(
+        "Renders hollow redaction outlines and a top-right legend on the page image; "
+        "writes ``redaction_overlay/{doc_base_name}_page{n}_redaction_overlay.jpg`` under OUTPUT_FOLDER "
+        "(scaled per REVIEW_OVERLAY_MAX_PIXELS, JPEG capped by REVIEW_OVERLAY_MAX_FILE_BYTES). "
+        "Uses ``tools.redaction_review.visualise_review_redaction_boxes``."
+    ),
+)
+def post_export_review_redaction_overlay(
+    body: AgentExportReviewRedactionOverlayRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    import pandas as pd
+
+    from tools.redaction_review import visualise_review_redaction_boxes
+
+    img_path = _path_must_be_allowed_file(body.page_image_path)
+    annotator: dict[str, Any] = {"image": img_path, "boxes": body.boxes}
+    review_df = (
+        pd.DataFrame(body.review_df_records)
+        if body.review_df_records
+        else pd.DataFrame()
+    )
+    out_folder_abs = os.path.realpath(
+        os.path.abspath(os.path.expanduser(str(OUTPUT_FOLDER)))
+    )
+    if not validate_path_safety(out_folder_abs):
+        raise HTTPException(status_code=400, detail="Unsafe OUTPUT_FOLDER path")
+    _must_be_under_allowed_roots(out_folder_abs, str(out_folder_abs))
+    try:
+        Path(out_folder_abs).mkdir(parents=True, exist_ok=True)
+    except OSError:
+        raise HTTPException(status_code=500, detail="Could not create OUTPUT_FOLDER")
+    out_folder = out_folder_abs
+
+    path = visualise_review_redaction_boxes(
+        annotator,
+        review_df=review_df,
+        output_folder=out_folder,
+        page_number=body.page_number,
+        doc_base_name=body.doc_base_name,
+        label_abbrev_chars=body.label_abbrev_chars,
+    )
+    if not path:
+        raise HTTPException(
+            status_code=500,
+            detail=(
+                "Could not produce overlay PNG (invalid image/boxes or write failed). "
+                "Ensure boxes are valid and the image loads."
+            ),
+        )
+    return AgentTaskResponse(
+        status="completed",
+        gradio_api_name="export_review_redaction_overlay",
+        task="export_review_redaction_overlay",
+        output_dir=out_folder,
+        input_dir="",
+        message="Redaction overlay PNG written",
+        output_paths=[path],
+    )
+
+
+@router.post(
+    "/export_review_page_ocr_visualisation",
+    response_model=AgentTaskResponse,
+    summary="export_review_page_ocr_visualisation (Agent API; Gradio api_name: page_ocr_review_image)",
+    description=(
+        "Renders a per-page OCR visualisation using tools.file_redaction.visualise_ocr_words_bounding_boxes; "
+        "writes under OUTPUT_FOLDER/review_ocr_visualisations/."
+    ),
+)
+def post_export_review_page_ocr_visualisation(
+    body: AgentExportReviewPageOcrVisualisationRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from PIL import Image
+
+    from tools.file_redaction import visualise_ocr_words_bounding_boxes
+
+    img_path = _path_must_be_allowed_file(body.page_image_path)
+
+    out_folder_abs = os.path.realpath(
+        os.path.abspath(os.path.expanduser(str(OUTPUT_FOLDER)))
+    )
+    if not validate_path_safety(out_folder_abs):
+        raise HTTPException(status_code=400, detail="Unsafe OUTPUT_FOLDER path")
+    _must_be_under_allowed_roots(out_folder_abs, str(out_folder_abs))
+    try:
+        Path(out_folder_abs).mkdir(parents=True, exist_ok=True)
+    except OSError:
+        raise HTTPException(status_code=500, detail="Could not create OUTPUT_FOLDER")
+    out_folder = out_folder_abs
+
+    safe_base = str(body.doc_base_name or "review")
+    image_name = f"{safe_base}_page{int(body.page_number)}.png"
+    log_paths: list[str] = []
+    try:
+        log_paths = visualise_ocr_words_bounding_boxes(
+            Image.open(img_path).convert("RGB"),
+            body.ocr_results,
+            image_name=image_name,
+            output_folder=out_folder,
+            visualisation_folder="review_ocr_visualisations",
+            add_legend=True,
+            log_files_output_paths=log_paths,
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e)) from e
+
+    if not log_paths:
+        raise HTTPException(
+            status_code=500,
+            detail="Could not produce OCR visualisation (invalid image/ocr_results or write failed).",
+        )
+    out_path = log_paths[-1]
+    return AgentTaskResponse(
+        status="completed",
+        gradio_api_name="export_review_page_ocr_visualisation",
+        task="export_review_page_ocr_visualisation",
+        output_dir=out_folder,
+        input_dir="",
+        message="OCR visualisation written",
+        output_paths=[out_path],
+    )
+
+
+def _gradio_only(api_name: str, detail: str) -> JSONResponse:
+    return JSONResponse(
+        status_code=501,
+        content={
+            "gradio_api_name": api_name,
+            "detail": detail,
+            "hint": (
+                "This flow is Gradio-session stateful. Call the named route on the "
+                "Gradio HTTP API, not /agent."
+            ),
+            "gradio_http": {
+                "discover_schema": "GET /gradio_api/info",
+                "start_call": f"POST /gradio_api/call/{api_name}",
+                "request_body_shape": '{"data": [<args in schema order>]}',
+                "poll": f"GET /gradio_api/call/{api_name}/{{event_id}}",
+            },
+            "gradio_client_notes": [
+                "Pass api_name explicitly; do not rely on inferring the endpoint from "
+                "Python function names (large Blocks apps will look ambiguous).",
+                "If predict() still cannot resolve the route, open GET /gradio_api/info "
+                "and use the numeric fn_index with gradio_client, or call the HTTP "
+                "endpoints directly.",
+                "The length of data must match the parameter list for this deployment; "
+                "copy order and types from /gradio_api/info.",
+            ],
+        },
+    )
+
+
+@router.post("/load_and_prepare_documents_or_data")
+def post_load_and_prepare_documents_or_data() -> JSONResponse:
+    return _gradio_only(
+        "load_and_prepare_documents_or_data",
+        "Preparation uses Gradio session state and prepare_image_or_pdf_with_efficient_ocr; no single CLI task.",
+    )
+
+
+@router.post(
+    "/apply_review_redactions",
+    response_model=AgentTaskResponse,
+    summary="apply_review_redactions (Gradio api_name)",
+    description=(
+        "Runs prepare_image_or_pdf_with_efficient_ocr([pdf, review_csv]) then "
+        "apply_redactions_to_review_df_and_files — same core pipeline as the Review tab, "
+        "without Gradio session state. Requires paths under allowed roots."
+    ),
+)
+def post_apply_review_redactions(
+    body: AgentApplyReviewRedactionsRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from tools.simplified_api import run_apply_review_redactions
+
+    pdf = _path_must_be_allowed_file(body.pdf_path)
+    csv = _path_must_be_allowed_file(body.review_csv_path)
+    out_dir: str | None = None
+    if body.output_dir is not None:
+        out_dir = _path_must_be_allowed_directory(body.output_dir, must_exist=False)
+    in_dir: str | None = None
+    if body.input_dir is not None:
+        in_dir = _path_must_be_allowed_directory(body.input_dir, must_exist=False)
+
+    try:
+        result = run_apply_review_redactions(
+            pdf_path=pdf,
+            review_csv_path=csv,
+            output_dir=out_dir,
+            input_dir=in_dir,
+            text_extract_method=body.text_extract_method,
+            efficient_ocr=body.efficient_ocr,
+        )
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e)) from e
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"apply_review_redactions failed: {e}",
+        ) from e
+
+    return AgentTaskResponse(
+        status="completed",
+        gradio_api_name="apply_review_redactions",
+        task="apply_review_redactions",
+        output_dir=result["output_dir"],
+        input_dir=result["input_dir"],
+        message=result["message"],
+        output_paths=result.get("output_paths"),
+    )
+
+
+@router.post(
+    "/verify_redaction_coverage",
+    response_model=AgentVerifyRedactionResponse,
+    summary="verify_redaction_coverage (Pass 1 programmatic QA)",
+)
+def post_verify_redaction_coverage(
+    body: AgentVerifyRedactionRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentVerifyRedactionResponse:
+    from tools.simplified_api import run_verify_redaction_coverage
+
+    review = _path_must_be_allowed_file(body.review_csv_path)
+    ocr_words = _path_must_be_allowed_file(body.ocr_words_csv_path)
+    redacted = None
+    if body.redacted_pdf_path:
+        redacted = _path_must_be_allowed_file(body.redacted_pdf_path)
+    try:
+        report, pruned_csv_path, prune_log = run_verify_redaction_coverage(
+            review_csv_path=review,
+            ocr_words_csv_path=ocr_words,
+            must_redact=body.must_redact,
+            must_not_redact=body.must_not_redact,
+            redacted_pdf_path=redacted,
+            total_pages=body.total_pages,
+            min_word_length=body.min_word_length,
+            sample_pixels=body.sample_pixels,
+            auto_prune_suspicious=body.auto_prune_suspicious,
+            pruned_output_path=body.pruned_output_path,
+        )
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e)) from e
+    except Exception as e:
+        raise HTTPException(
+            status_code=500, detail=f"verify_redaction_coverage failed: {e}"
+        ) from e
+    return AgentVerifyRedactionResponse(
+        status="completed",
+        coverage_pass=bool(report.get("pass_strict", report.get("pass"))),
+        coverage_pass_strict=bool(report.get("pass_strict", report.get("pass"))),
+        coverage_pass_with_cleanup=bool(report.get("pass_with_cleanup")),
+        pruned_csv_path=pruned_csv_path,
+        prune_log=prune_log,
+        report=report,
+    )
+
+
+@router.post(
+    "/word_level_ocr_text_search",
+    response_model=AgentWordLevelOcrSearchResponse,
+    summary="word_level_ocr_text_search (headless OCR CSV search)",
+)
+def post_word_level_ocr_text_search(
+    body: AgentWordLevelOcrSearchRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentWordLevelOcrSearchResponse:
+    from tools.simplified_api import run_word_level_ocr_text_search_api
+
+    ocr_words = _path_must_be_allowed_file(body.ocr_words_csv_path)
+    review = None
+    if body.review_csv_path:
+        review = _path_must_be_allowed_file(body.review_csv_path)
+    try:
+        result = run_word_level_ocr_text_search_api(
+            ocr_words_csv_path=ocr_words,
+            search_text=body.search_text,
+            similarity_threshold=body.similarity_threshold,
+            use_regex=body.use_regex,
+            review_csv_path=review,
+        )
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e)) from e
+    except Exception as e:
+        raise HTTPException(
+            status_code=500, detail=f"word_level_ocr_text_search failed: {e}"
+        ) from e
+    return AgentWordLevelOcrSearchResponse(status="completed", result=result)
+
+
+@router.get("/operations")
+def list_operations() -> dict[str, Any]:
+    return {
+        "gradio_api_names": list(GRADIO_API_NAMES),
+        "gradio_session_state_endpoints": {
+            "description": (
+                "These api_name values are exposed on the Gradio HTTP API but return "
+                "501 on /agent because they depend on in-memory Gradio state."
+            ),
+            "discover_schema": "GET /gradio_api/info",
+            "call_pattern": 'POST /gradio_api/call/<api_name> with JSON body {"data": [...]}',
+            "names": [
+                "load_and_prepare_documents_or_data",
+            ],
+        },
+        "routes": [
+            {
+                "gradio_api_name": "redact_document",
+                "method": "POST",
+                "path": "/agent/redact_document",
+                "implementation": "cli_redact task redact",
+                "notes": {
+                    "ocr_method": [
+                        "Local OCR",
+                        "AWS Textract",
+                        "Local text",
+                    ],
+                    "chosen_local_ocr_model_override": LOCAL_OCR_MODEL_OPTIONS,
+                    "pii_detector_recommended": [
+                        LOCAL_PII_OPTION,
+                        AWS_PII_OPTION,
+                        AWS_LLM_PII_OPTION,
+                        INFERENCE_SERVER_PII_OPTION,
+                        LOCAL_TRANSFORMERS_LLM_PII_OPTION,
+                        "None",
+                    ],
+                },
+            },
+            {
+                "gradio_api_name": "redact_data",
+                "method": "POST",
+                "path": "/agent/redact_data",
+                "implementation": "cli_redact task redact",
+                "notes": {
+                    "ocr_method": [
+                        "Local OCR",
+                        "AWS Textract",
+                        "Local text",
+                    ],
+                    "chosen_local_ocr_model_override": LOCAL_OCR_MODEL_OPTIONS,
+                    "pii_detector_recommended": [
+                        LOCAL_PII_OPTION,
+                        AWS_PII_OPTION,
+                        AWS_LLM_PII_OPTION,
+                        INFERENCE_SERVER_PII_OPTION,
+                        LOCAL_TRANSFORMERS_LLM_PII_OPTION,
+                        "None",
+                    ],
+                },
+            },
+            {
+                "gradio_api_name": "find_duplicate_pages",
+                "method": "POST",
+                "path": "/agent/find_duplicate_pages",
+                "implementation": "cli_redact deduplicate pages",
+            },
+            {
+                "gradio_api_name": "find_duplicate_tabular",
+                "method": "POST",
+                "path": "/agent/find_duplicate_tabular",
+                "implementation": "cli_redact deduplicate tabular",
+            },
+            {
+                "gradio_api_name": "summarise_document",
+                "method": "POST",
+                "path": "/agent/summarise_document",
+                "implementation": "cli_redact task summarise",
+            },
+            {
+                "gradio_api_name": "combine_review_pdfs",
+                "method": "POST",
+                "path": "/agent/combine_review_pdfs",
+                "implementation": "cli_redact combine_review_pdfs",
+            },
+            {
+                "gradio_api_name": "export_review_redaction_overlay",
+                "method": "POST",
+                "path": "/agent/export_review_redaction_overlay",
+                "implementation": "visualise_review_redaction_boxes",
+            },
+            {
+                "gradio_api_name": "export_review_page_ocr_visualisation",
+                "method": "POST",
+                "path": "/agent/export_review_page_ocr_visualisation",
+                "implementation": "visualise_ocr_words_bounding_boxes",
+            },
+            {
+                "gradio_api_name": "combine_review_csvs",
+                "method": "POST",
+                "path": "/agent/combine_review_csvs",
+                "implementation": "helper merge_csv_files",
+            },
+            {
+                "gradio_api_name": "load_and_prepare_documents_or_data",
+                "method": "POST",
+                "path": "/agent/load_and_prepare_documents_or_data",
+                "implementation": "not_implemented_http",
+            },
+            {
+                "gradio_api_name": "apply_review_redactions",
+                "method": "POST",
+                "path": "/agent/apply_review_redactions",
+                "implementation": "tools.simplified_api.run_apply_review_redactions",
+            },
+            {
+                "gradio_api_name": "verify_redaction_coverage",
+                "method": "POST",
+                "path": "/agent/verify_redaction_coverage",
+                "implementation": "tools.verify_redaction_coverage.verify_redaction_coverage",
+                "notes": {
+                    "purpose": "Pass 1 programmatic QA — pass_strict (policy), pass_with_cleanup (+ suspicious rows), optional prune and text/pixel checks.",
+                    "must_redact": "list of regex strings",
+                    "must_not_redact": "list of regex strings",
+                    "auto_prune_suspicious": "remove short OCR-fragment rows before reporting",
+                    "pages_flagged_for_vlm": "policy/visual failures only",
+                    "pages_needing_csv_cleanup": "suspicious rows — prune, not VLM",
+                    "leak_likely_causes": "per-page hints when text_layer_leaks (coord_not_normalized, missing_page_boxes, etc.) — not a broken /review_apply",
+                },
+            },
+            {
+                "gradio_api_name": "word_level_ocr_text_search",
+                "method": "POST",
+                "path": "/agent/word_level_ocr_text_search",
+                "implementation": "tools.verify_redaction_coverage.run_word_level_ocr_text_search",
+            },
+        ],
+    }
+
+
+@router.get("/health")
+def agent_health() -> dict[str, str]:
+    return {"status": "ok", "service": "agent"}

cdk/app.py

@@ -0,0 +1,85 @@
+import os
+
+from aws_cdk import App, Environment
+from cdk_config import (
+    AWS_ACCOUNT_ID,
+    AWS_REGION,
+    CDK_CONTEXT_FILE,
+    RUN_USEAST_STACK,
+    USE_CLOUDFRONT,
+)
+from cdk_functions import (
+    create_basic_config_env,
+    load_context_from_file,
+    log_aws_credential_context,
+    purge_cdk_lookup_context,
+)
+from cdk_stack import CdkStack, CdkStackCloudfront  # , CdkStackMain
+from check_resources import CONTEXT_FILE, check_and_set_context
+
+# Initialize the CDK app
+app = App()
+
+log_aws_credential_context(
+    expected_account_id=AWS_ACCOUNT_ID,
+    expected_region=AWS_REGION,
+)
+
+# Drop stale CDK lookup cache entries (require bootstrap lookup role in target account).
+purge_cdk_lookup_context(CDK_CONTEXT_FILE)
+
+# --- Pre-check context (boto3) — written to precheck.context.json, NOT cdk.context.json ---
+print(f"Pre-check context file: {CONTEXT_FILE}")
+print(f"CDK lookup cache file: {CDK_CONTEXT_FILE}")
+if os.path.basename(CONTEXT_FILE.replace("\\", "/")) == os.path.basename(
+    CDK_CONTEXT_FILE.replace("\\", "/")
+):
+    raise RuntimeError(
+        f"CONTEXT_FILE and CDK_CONTEXT_FILE must differ (got '{CONTEXT_FILE}' for both). "
+        "Set CONTEXT_FILE=precheck.context.json in config/cdk_config.env."
+    )
+
+print("Running pre-check script to generate application context...")
+try:
+    check_and_set_context()
+    if not os.path.exists(CONTEXT_FILE):
+        raise RuntimeError(
+            f"check_and_set_context() finished, but {CONTEXT_FILE} was not created."
+        )
+    print(f"Context generated successfully at {CONTEXT_FILE}.")
+except Exception as e:
+    raise RuntimeError(f"Failed to generate context via check_and_set_context(): {e}")
+
+# Pre-check must not repopulate CDK lookup keys; purge again if paths were ever shared.
+purge_cdk_lookup_context(CDK_CONTEXT_FILE)
+
+if os.path.exists(CONTEXT_FILE):
+    load_context_from_file(app, CONTEXT_FILE)
+else:
+    raise RuntimeError(f"Could not find {CONTEXT_FILE}.")
+
+create_basic_config_env("config")
+
+aws_env_regional = Environment(account=AWS_ACCOUNT_ID, region=AWS_REGION)
+
+regional_stack = CdkStack(
+    app, "RedactionStack", env=aws_env_regional, cross_region_references=True
+)
+
+if USE_CLOUDFRONT == "True" and RUN_USEAST_STACK == "True":
+    aws_env_us_east_1 = Environment(account=AWS_ACCOUNT_ID, region="us-east-1")
+
+    cloudfront_stack = CdkStackCloudfront(
+        app,
+        "RedactionStackCloudfront",
+        env=aws_env_us_east_1,
+        alb_arn=regional_stack.params["alb_arn_output"],
+        alb_sec_group_id=regional_stack.params["alb_security_group_id"],
+        alb_dns_name=regional_stack.params["alb_dns_name"],
+        cross_region_references=True,
+    )
+
+# CDK CLI invokes this script and expects a cloud assembly in cdk.out.
+# Without app.synth(), Python defines constructs but never writes manifest.json
+# (ENOENT on deploy). See: https://github.com/aws/aws-cdk/issues/11023
+app.synth()

cdk/cdk.json.example

@@ -0,0 +1,7 @@
+{
+  "app": "python app.py",
+  "output": "cdk.out",
+  "context": {
+    "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": false
+  }
+}

cdk/cdk_config.py

@@ -0,0 +1,371 @@
+import os
+import tempfile
+
+from dotenv import load_dotenv
+
+# Set or retrieve configuration variables for CDK redaction deployment
+
+
+def convert_string_to_boolean(value: str) -> bool:
+    """Convert string to boolean, handling various formats."""
+    if isinstance(value, bool):
+        return value
+    elif value in ["True", "1", "true", "TRUE"]:
+        return True
+    elif value in ["False", "0", "false", "FALSE"]:
+        return False
+    else:
+        raise ValueError(f"Invalid boolean value: {value}")
+
+
+def get_or_create_env_var(var_name: str, default_value: str, print_val: bool = False):
+    """
+    Get an environmental variable, and set it to a default value if it doesn't exist
+    """
+    # Get the environment variable if it exists
+    value = os.environ.get(var_name)
+
+    # If it doesn't exist, set the environment variable to the default value
+    if value is None:
+        os.environ[var_name] = default_value
+        value = default_value
+
+    if print_val is True:
+        print(f"The value of {var_name} is {value}")
+
+    return value
+
+
+def ensure_folder_exists(output_folder: str):
+    """Checks if the specified folder exists, creates it if not."""
+
+    if not os.path.exists(output_folder):
+        # Create the folder if it doesn't exist
+        os.makedirs(output_folder, exist_ok=True)
+        print(f"Created the {output_folder} folder.")
+    else:
+        print(f"The {output_folder} folder already exists.")
+
+
+def add_folder_to_path(folder_path: str):
+    """
+    Check if a folder exists on your system. If so, get the absolute path and then add it to the system Path variable if it doesn't already exist. Function is only relevant for locally-created executable files based on this app (when using pyinstaller it creates a _internal folder that contains tesseract and poppler. These need to be added to the system path to enable the app to run)
+    """
+
+    if os.path.exists(folder_path) and os.path.isdir(folder_path):
+        print(folder_path, "folder exists.")
+
+        # Resolve relative path to absolute path
+        absolute_path = os.path.abspath(folder_path)
+
+        current_path = os.environ["PATH"]
+        if absolute_path not in current_path.split(os.pathsep):
+            full_path_extension = absolute_path + os.pathsep + current_path
+            os.environ["PATH"] = full_path_extension
+            # print(f"Updated PATH with: ", full_path_extension)
+        else:
+            print(f"Directory {folder_path} already exists in PATH.")
+    else:
+        print(f"Folder not found at {folder_path} - not added to PATH")
+
+
+###
+# LOAD CONFIG FROM ENV FILE
+###
+CONFIG_FOLDER = get_or_create_env_var("CONFIG_FOLDER", "config/")
+
+ensure_folder_exists(CONFIG_FOLDER)
+
+# If you have an aws_config env file in the config folder, you can load in app variables this way, e.g. 'config/cdk_config.env'
+CDK_CONFIG_PATH = get_or_create_env_var(
+    "CDK_CONFIG_PATH", "config/cdk_config.env"
+)  # e.g. config/cdk_config.env
+
+if CDK_CONFIG_PATH:
+    if os.path.exists(CDK_CONFIG_PATH):
+        print(f"Loading CDK variables from config file {CDK_CONFIG_PATH}")
+        load_dotenv(CDK_CONFIG_PATH)
+    else:
+        print("CDK config file not found at location:", CDK_CONFIG_PATH)
+
+###
+# AWS OPTIONS
+###
+AWS_REGION = get_or_create_env_var("AWS_REGION", "")
+AWS_ACCOUNT_ID = get_or_create_env_var("AWS_ACCOUNT_ID", "")
+
+###
+# CDK OPTIONS
+###
+CDK_PREFIX = get_or_create_env_var("CDK_PREFIX", "")
+_precheck_context_file = get_or_create_env_var("CONTEXT_FILE", "precheck.context.json")
+# Never write boto3 pre-check output into CDK's lookup cache file (causes stale
+# vpc-provider / load-balancer entries and wrong-account lookup validation errors).
+if os.path.basename(_precheck_context_file.replace("\\", "/")) == "cdk.context.json":
+    print(
+        "WARNING: CONTEXT_FILE must not be 'cdk.context.json' (that file is CDK's "
+        "lookup cache). Using 'precheck.context.json' instead. Update "
+        "config/cdk_config.env and remove CONTEXT_FILE=cdk.context.json if set."
+    )
+    _precheck_context_file = "precheck.context.json"
+CONTEXT_FILE = _precheck_context_file
+CDK_CONTEXT_FILE = get_or_create_env_var("CDK_CONTEXT_FILE", "cdk.context.json")
+CDK_FOLDER = get_or_create_env_var(
+    "CDK_FOLDER", ""
+)  # FULL_PATH_TO_CDK_FOLDER_HERE (with forward slash)
+RUN_USEAST_STACK = get_or_create_env_var("RUN_USEAST_STACK", "False")
+
+### VPC and connections
+VPC_NAME = get_or_create_env_var("VPC_NAME", "")
+NEW_VPC_DEFAULT_NAME = get_or_create_env_var("NEW_VPC_DEFAULT_NAME", f"{CDK_PREFIX}vpc")
+NEW_VPC_CIDR = get_or_create_env_var("NEW_VPC_CIDR", "")  # "10.0.0.0/24"
+
+
+EXISTING_IGW_ID = get_or_create_env_var("EXISTING_IGW_ID", "")
+SINGLE_NAT_GATEWAY_ID = get_or_create_env_var("SINGLE_NAT_GATEWAY_ID", "")
+
+### SUBNETS / ROUTE TABLES / NAT GATEWAY
+PUBLIC_SUBNETS_TO_USE = get_or_create_env_var(
+    "PUBLIC_SUBNETS_TO_USE", ""
+)  # e.g. ['PublicSubnet1', 'PublicSubnet2']
+PUBLIC_SUBNET_CIDR_BLOCKS = get_or_create_env_var(
+    "PUBLIC_SUBNET_CIDR_BLOCKS", ""
+)  # e.g. ["10.0.1.0/24", "10.0.2.0/24"]
+PUBLIC_SUBNET_AVAILABILITY_ZONES = get_or_create_env_var(
+    "PUBLIC_SUBNET_AVAILABILITY_ZONES", ""
+)  # e.g. ["eu-east-1b", "eu-east1b"]
+
+PRIVATE_SUBNETS_TO_USE = get_or_create_env_var(
+    "PRIVATE_SUBNETS_TO_USE", ""
+)  # e.g. ['PrivateSubnet1', 'PrivateSubnet2']
+PRIVATE_SUBNET_CIDR_BLOCKS = get_or_create_env_var(
+    "PRIVATE_SUBNET_CIDR_BLOCKS", ""
+)  # e.g. ["10.0.1.0/24", "10.0.2.0/24"]
+PRIVATE_SUBNET_AVAILABILITY_ZONES = get_or_create_env_var(
+    "PRIVATE_SUBNET_AVAILABILITY_ZONES", ""
+)  # e.g. ["eu-east-1b", "eu-east1b"]
+
+ROUTE_TABLE_BASE_NAME = get_or_create_env_var(
+    "ROUTE_TABLE_BASE_NAME", f"{CDK_PREFIX}PrivateRouteTable"
+)
+NAT_GATEWAY_EIP_NAME = get_or_create_env_var(
+    "NAT_GATEWAY_EIP_NAME", f"{CDK_PREFIX}NatGatewayEip"
+)
+NAT_GATEWAY_NAME = get_or_create_env_var("NAT_GATEWAY_NAME", f"{CDK_PREFIX}NatGateway")
+
+# IAM roles
+AWS_MANAGED_TASK_ROLES_LIST = get_or_create_env_var(
+    "AWS_MANAGED_TASK_ROLES_LIST",
+    '["AmazonCognitoReadOnly", "service-role/AmazonECSTaskExecutionRolePolicy", "AmazonS3FullAccess", "AmazonTextractFullAccess", "ComprehendReadOnly", "AmazonDynamoDBFullAccess", "service-role/AWSAppSyncPushToCloudWatchLogs", "AmazonBedrockFullAccess"]',
+)
+POLICY_FILE_LOCATIONS = get_or_create_env_var(
+    "POLICY_FILE_LOCATIONS", ""
+)  # e.g. '["config/sts_permissions.json"]'
+POLICY_FILE_ARNS = get_or_create_env_var("POLICY_FILE_ARNS", "")
+
+# GITHUB REPO
+GITHUB_REPO_USERNAME = get_or_create_env_var("GITHUB_REPO_USERNAME", "seanpedrick-case")
+GITHUB_REPO_NAME = get_or_create_env_var("GITHUB_REPO_NAME", "doc_redaction")
+GITHUB_REPO_BRANCH = get_or_create_env_var("GITHUB_REPO_BRANCH", "main")
+
+### CODEBUILD
+CODEBUILD_ROLE_NAME = get_or_create_env_var(
+    "CODEBUILD_ROLE_NAME", f"{CDK_PREFIX}CodeBuildRole"
+)
+CODEBUILD_PROJECT_NAME = get_or_create_env_var(
+    "CODEBUILD_PROJECT_NAME", f"{CDK_PREFIX}CodeBuildProject"
+)
+
+### ECR
+ECR_REPO_NAME = get_or_create_env_var(
+    "ECR_REPO_NAME", "doc-redaction"
+)  # Beware - cannot have underscores and must be lower case
+ECR_CDK_REPO_NAME = get_or_create_env_var(
+    "ECR_CDK_REPO_NAME", f"{CDK_PREFIX}{ECR_REPO_NAME}".lower()
+)
+
+### S3
+S3_LOG_CONFIG_BUCKET_NAME = get_or_create_env_var(
+    "S3_LOG_CONFIG_BUCKET_NAME", f"{CDK_PREFIX}s3-logs".lower()
+)  # S3 bucket names need to be lower case
+S3_OUTPUT_BUCKET_NAME = get_or_create_env_var(
+    "S3_OUTPUT_BUCKET_NAME", f"{CDK_PREFIX}s3-output".lower()
+)
+
+### KMS KEYS FOR S3 AND SECRETS MANAGER
+USE_CUSTOM_KMS_KEY = get_or_create_env_var("USE_CUSTOM_KMS_KEY", "1")
+CUSTOM_KMS_KEY_NAME = get_or_create_env_var(
+    "CUSTOM_KMS_KEY_NAME", f"alias/{CDK_PREFIX}kms-key".lower()
+)
+
+### ECS
+FARGATE_TASK_DEFINITION_NAME = get_or_create_env_var(
+    "FARGATE_TASK_DEFINITION_NAME", f"{CDK_PREFIX}FargateTaskDefinition"
+)
+TASK_DEFINITION_FILE_LOCATION = get_or_create_env_var(
+    "TASK_DEFINITION_FILE_LOCATION", CDK_FOLDER + CONFIG_FOLDER + "task_definition.json"
+)
+
+CLUSTER_NAME = get_or_create_env_var("CLUSTER_NAME", f"{CDK_PREFIX}Cluster")
+ECS_SERVICE_NAME = get_or_create_env_var("ECS_SERVICE_NAME", f"{CDK_PREFIX}ECSService")
+ECS_TASK_ROLE_NAME = get_or_create_env_var(
+    "ECS_TASK_ROLE_NAME", f"{CDK_PREFIX}TaskRole"
+)
+ECS_TASK_EXECUTION_ROLE_NAME = get_or_create_env_var(
+    "ECS_TASK_EXECUTION_ROLE_NAME", f"{CDK_PREFIX}ExecutionRole"
+)
+ECS_SECURITY_GROUP_NAME = get_or_create_env_var(
+    "ECS_SECURITY_GROUP_NAME", f"{CDK_PREFIX}SecurityGroupECS"
+)
+ECS_LOG_GROUP_NAME = get_or_create_env_var(
+    "ECS_LOG_GROUP_NAME", f"/ecs/{ECS_SERVICE_NAME}-logs".lower()
+)
+
+ECS_TASK_CPU_SIZE = get_or_create_env_var("ECS_TASK_CPU_SIZE", "1024")
+ECS_TASK_MEMORY_SIZE = get_or_create_env_var("ECS_TASK_MEMORY_SIZE", "4096")
+ECS_USE_FARGATE_SPOT = get_or_create_env_var("USE_FARGATE_SPOT", "False")
+ECS_READ_ONLY_FILE_SYSTEM = get_or_create_env_var("ECS_READ_ONLY_FILE_SYSTEM", "True")
+
+### Cognito
+COGNITO_USER_POOL_NAME = get_or_create_env_var(
+    "COGNITO_USER_POOL_NAME", f"{CDK_PREFIX}UserPool"
+)
+COGNITO_USER_POOL_CLIENT_NAME = get_or_create_env_var(
+    "COGNITO_USER_POOL_CLIENT_NAME", f"{CDK_PREFIX}UserPoolClient"
+)
+COGNITO_USER_POOL_CLIENT_SECRET_NAME = get_or_create_env_var(
+    "COGNITO_USER_POOL_CLIENT_SECRET_NAME", f"{CDK_PREFIX}ParamCognitoSecret"
+)
+COGNITO_USER_POOL_DOMAIN_PREFIX = get_or_create_env_var(
+    "COGNITO_USER_POOL_DOMAIN_PREFIX", "redaction-app-domain"
+)  # Should change this to something unique or you'll probably hit an error
+
+COGNITO_REFRESH_TOKEN_VALIDITY = int(
+    get_or_create_env_var("COGNITO_REFRESH_TOKEN_VALIDITY", "480")
+)  # Minutes
+COGNITO_ID_TOKEN_VALIDITY = int(
+    get_or_create_env_var("COGNITO_ID_TOKEN_VALIDITY", "60")
+)  # Minutes
+COGNITO_ACCESS_TOKEN_VALIDITY = int(
+    get_or_create_env_var("COGNITO_ACCESS_TOKEN_VALIDITY", "60")
+)  # Minutes
+
+# Application load balancer
+ALB_NAME = get_or_create_env_var(
+    "ALB_NAME", f"{CDK_PREFIX}Alb"[-32:]
+)  # Application load balancer name can be max 32 characters, so taking the last 32 characters of the suggested name
+ALB_NAME_SECURITY_GROUP_NAME = get_or_create_env_var(
+    "ALB_SECURITY_GROUP_NAME", f"{CDK_PREFIX}SecurityGroupALB"
+)
+ALB_TARGET_GROUP_NAME = get_or_create_env_var(
+    "ALB_TARGET_GROUP_NAME", f"{CDK_PREFIX}-tg"[-32:]
+)  # Max 32 characters
+EXISTING_LOAD_BALANCER_ARN = get_or_create_env_var("EXISTING_LOAD_BALANCER_ARN", "")
+EXISTING_LOAD_BALANCER_DNS = get_or_create_env_var(
+    "EXISTING_LOAD_BALANCER_DNS", "placeholder_load_balancer_dns.net"
+)
+
+## CLOUDFRONT
+USE_CLOUDFRONT = get_or_create_env_var("USE_CLOUDFRONT", "True")
+CLOUDFRONT_PREFIX_LIST_ID = get_or_create_env_var(
+    "CLOUDFRONT_PREFIX_LIST_ID", "pl-93a247fa"
+)
+CLOUDFRONT_GEO_RESTRICTION = get_or_create_env_var(
+    "CLOUDFRONT_GEO_RESTRICTION", ""
+)  # A country that Cloudfront restricts access to. See here: https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/georestrictions.html
+CLOUDFRONT_DISTRIBUTION_NAME = get_or_create_env_var(
+    "CLOUDFRONT_DISTRIBUTION_NAME", f"{CDK_PREFIX}CfDist"
+)
+CLOUDFRONT_DOMAIN = get_or_create_env_var(
+    "CLOUDFRONT_DOMAIN", "cloudfront_placeholder.net"
+)
+
+
+# Certificate for Application load balancer (optional, for HTTPS and logins through the ALB)
+ACM_SSL_CERTIFICATE_ARN = get_or_create_env_var("ACM_SSL_CERTIFICATE_ARN", "")
+SSL_CERTIFICATE_DOMAIN = get_or_create_env_var(
+    "SSL_CERTIFICATE_DOMAIN", ""
+)  # e.g. example.com or www.example.com
+
+# This should be the CloudFront domain, the domain linked to your ACM certificate, or the DNS of your application load balancer in console afterwards
+if USE_CLOUDFRONT == "True":
+    COGNITO_REDIRECTION_URL = get_or_create_env_var(
+        "COGNITO_REDIRECTION_URL", "https://" + CLOUDFRONT_DOMAIN
+    )
+elif SSL_CERTIFICATE_DOMAIN:
+    COGNITO_REDIRECTION_URL = get_or_create_env_var(
+        "COGNITO_REDIRECTION_URL", "https://" + SSL_CERTIFICATE_DOMAIN
+    )
+else:
+    COGNITO_REDIRECTION_URL = get_or_create_env_var(
+        "COGNITO_REDIRECTION_URL", "https://" + EXISTING_LOAD_BALANCER_DNS
+    )
+
+# Custom headers e.g. if routing traffic through Cloudfront
+CUSTOM_HEADER = get_or_create_env_var(
+    "CUSTOM_HEADER", ""
+)  # Retrieving or setting CUSTOM_HEADER
+CUSTOM_HEADER_VALUE = get_or_create_env_var(
+    "CUSTOM_HEADER_VALUE", ""
+)  # Retrieving or setting CUSTOM_HEADER_VALUE
+
+# Firewall on top of load balancer
+LOAD_BALANCER_WEB_ACL_NAME = get_or_create_env_var(
+    "LOAD_BALANCER_WEB_ACL_NAME", f"{CDK_PREFIX}alb-web-acl"
+)
+
+# Firewall on top of CloudFront
+WEB_ACL_NAME = get_or_create_env_var("WEB_ACL_NAME", f"{CDK_PREFIX}cloudfront-web-acl")
+
+###
+# File I/O options
+###
+
+OUTPUT_FOLDER = get_or_create_env_var("GRADIO_OUTPUT_FOLDER", "output/")  # 'output/'
+INPUT_FOLDER = get_or_create_env_var("GRADIO_INPUT_FOLDER", "input/")  # 'input/'
+
+# Allow for files to be saved in a temporary folder for increased security in some instances
+if OUTPUT_FOLDER == "TEMP" or INPUT_FOLDER == "TEMP":
+    # Create a temporary directory
+    with tempfile.TemporaryDirectory() as temp_dir:
+        print(f"Temporary directory created at: {temp_dir}")
+
+        if OUTPUT_FOLDER == "TEMP":
+            OUTPUT_FOLDER = temp_dir + "/"
+        if INPUT_FOLDER == "TEMP":
+            INPUT_FOLDER = temp_dir + "/"
+
+###
+# LOGGING OPTIONS
+###
+
+SAVE_LOGS_TO_CSV = get_or_create_env_var("SAVE_LOGS_TO_CSV", "True")
+
+### DYNAMODB logs. Whether to save to DynamoDB, and the headers of the table
+SAVE_LOGS_TO_DYNAMODB = get_or_create_env_var("SAVE_LOGS_TO_DYNAMODB", "True")
+ACCESS_LOG_DYNAMODB_TABLE_NAME = get_or_create_env_var(
+    "ACCESS_LOG_DYNAMODB_TABLE_NAME", f"{CDK_PREFIX}dynamodb-access-logs".lower()
+)
+FEEDBACK_LOG_DYNAMODB_TABLE_NAME = get_or_create_env_var(
+    "FEEDBACK_LOG_DYNAMODB_TABLE_NAME", f"{CDK_PREFIX}dynamodb-feedback-logs".lower()
+)
+USAGE_LOG_DYNAMODB_TABLE_NAME = get_or_create_env_var(
+    "USAGE_LOG_DYNAMODB_TABLE_NAME", f"{CDK_PREFIX}dynamodb-usage-logs".lower()
+)
+
+###
+# REDACTION OPTIONS
+###
+
+# Get some environment variables and Launch the Gradio app
+COGNITO_AUTH = get_or_create_env_var("COGNITO_AUTH", "0")
+
+GRADIO_SERVER_PORT = int(get_or_create_env_var("GRADIO_SERVER_PORT", "7860"))
+
+###
+# WHOLE DOCUMENT API OPTIONS
+###
+
+DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS = get_or_create_env_var(
+    "DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS", "7"
+)  # How many days into the past should whole document Textract jobs be displayed? After that, the data is not deleted from the Textract jobs csv, but it is just filtered out. Included to align with S3 buckets where the file outputs will be automatically deleted after X days.

cdk/cdk_functions.py

@@ -0,0 +1,1665 @@
+import ipaddress
+import json
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+import boto3
+import pandas as pd
+from aws_cdk import App, CfnOutput, CfnTag, Tags
+from aws_cdk import aws_cognito as cognito
+from aws_cdk import aws_ec2 as ec2
+from aws_cdk import aws_elasticloadbalancingv2 as elb
+from aws_cdk import aws_elasticloadbalancingv2_actions as elb_act
+from aws_cdk import aws_iam as iam
+from aws_cdk import aws_wafv2 as wafv2
+from botocore.exceptions import ClientError, NoCredentialsError
+from cdk_config import (
+    ACCESS_LOG_DYNAMODB_TABLE_NAME,
+    AWS_REGION,
+    FEEDBACK_LOG_DYNAMODB_TABLE_NAME,
+    NAT_GATEWAY_EIP_NAME,
+    POLICY_FILE_LOCATIONS,
+    PRIVATE_SUBNET_AVAILABILITY_ZONES,
+    PRIVATE_SUBNET_CIDR_BLOCKS,
+    PRIVATE_SUBNETS_TO_USE,
+    PUBLIC_SUBNET_AVAILABILITY_ZONES,
+    PUBLIC_SUBNET_CIDR_BLOCKS,
+    PUBLIC_SUBNETS_TO_USE,
+    S3_LOG_CONFIG_BUCKET_NAME,
+    S3_OUTPUT_BUCKET_NAME,
+    USAGE_LOG_DYNAMODB_TABLE_NAME,
+)
+from constructs import Construct
+from dotenv import set_key
+
+# CDK CLI stores lookup-provider results under these key prefixes in cdk.context.json.
+_CDK_LOOKUP_CONTEXT_PREFIXES = (
+    "vpc-provider:",
+    "load-balancer:",
+    "availability-zones:",
+    "hosted-zone:",
+    "security-group:",
+    "key-provider:",
+    "ami:",
+)
+
+
+def purge_cdk_lookup_context(file_path: str) -> int:
+    """Remove stale CDK lookup cache entries that require the bootstrap lookup role."""
+    if not os.path.exists(file_path):
+        return 0
+    with open(file_path, "r", encoding="utf-8") as f:
+        context_data = json.load(f)
+    cleaned = {
+        key: value
+        for key, value in context_data.items()
+        if not key.startswith(_CDK_LOOKUP_CONTEXT_PREFIXES)
+    }
+    removed = len(context_data) - len(cleaned)
+    if removed:
+        with open(file_path, "w", encoding="utf-8") as f:
+            json.dump(cleaned, f, indent=2)
+        print(f"Removed {removed} stale CDK lookup context key(s) from {file_path}.")
+    return removed
+
+
+def log_aws_credential_context(
+    expected_account_id: Optional[str] = None,
+    expected_region: Optional[str] = None,
+) -> Dict[str, Any]:
+    """
+    Print the active AWS identity and non-secret credential hints for CDK debugging.
+
+    Helps distinguish SSO/assumed-role sessions from long-lived access keys in
+    ~/.aws/credentials or environment variables.
+    """
+    profile = os.environ.get("AWS_PROFILE") or "(not set — using default profile chain)"
+    default_region = (
+        os.environ.get("AWS_REGION")
+        or os.environ.get("AWS_DEFAULT_REGION")
+        or "(not set in environment)"
+    )
+    env_access_key_set = bool(os.environ.get("AWS_ACCESS_KEY_ID"))
+    env_secret_key_set = bool(os.environ.get("AWS_SECRET_ACCESS_KEY"))
+    env_session_token_set = bool(os.environ.get("AWS_SESSION_TOKEN"))
+
+    print("\n--- AWS credential context (CDK / boto3) ---")
+    print(f"AWS_PROFILE: {profile}")
+    print(f"AWS_REGION / AWS_DEFAULT_REGION (env): {default_region}")
+    print(
+        "Environment credential variables: "
+        f"AWS_ACCESS_KEY_ID={'set' if env_access_key_set else 'not set'}, "
+        f"AWS_SECRET_ACCESS_KEY={'set' if env_secret_key_set else 'not set'}, "
+        f"AWS_SESSION_TOKEN={'set' if env_session_token_set else 'not set'}"
+    )
+    if expected_account_id:
+        print(f"Configured CDK target account (AWS_ACCOUNT_ID): {expected_account_id}")
+    if expected_region:
+        print(f"Configured CDK target region (AWS_REGION): {expected_region}")
+
+    session = boto3.Session()
+    active_profile = session.profile_name or "(default)"
+    print(f"boto3 session profile: {active_profile}")
+    print(f"boto3 session region: {session.region_name or '(not set)'}")
+
+    credentials = session.get_credentials()
+    credential_summary: Dict[str, Any] = {
+        "profile": profile,
+        "session_profile": active_profile,
+    }
+
+    if credentials is None:
+        print("WARNING: No AWS credentials found in the default provider chain.")
+        print("--- End AWS credential context ---\n")
+        credential_summary["error"] = "no_credentials"
+        return credential_summary
+
+    frozen = credentials.get_frozen_credentials()
+    access_key = frozen.access_key or ""
+    access_key_prefix = (access_key[:4] + "...") if len(access_key) >= 4 else "(none)"
+    credential_summary["access_key_prefix"] = access_key_prefix
+
+    if env_access_key_set:
+        credential_source = "environment variables (highest precedence)"
+    elif access_key.startswith("AKIA"):
+        credential_source = "long-lived access key (likely ~/.aws/credentials [default] or named profile)"
+    elif access_key.startswith("ASIA"):
+        credential_source = "temporary credentials (SSO, assumed role, or STS session)"
+    else:
+        credential_source = (
+            "resolved credentials (source could not be classified from key prefix)"
+        )
+
+    print(f"Inferred credential type: {credential_source}")
+    credential_summary["inferred_credential_type"] = credential_source
+
+    if env_access_key_set and profile != "(not set — using default profile chain)":
+        print(
+            "NOTE: AWS_ACCESS_KEY_ID is set in the environment, so it overrides "
+            f"profile '{profile}' and SSO."
+        )
+
+    try:
+        sts = session.client("sts", region_name=session.region_name or expected_region)
+        identity = sts.get_caller_identity()
+    except (ClientError, NoCredentialsError) as exc:
+        print(f"WARNING: sts:GetCallerIdentity failed: {exc}")
+        print("--- End AWS credential context ---\n")
+        credential_summary["error"] = str(exc)
+        return credential_summary
+
+    account = identity.get("Account", "")
+    arn = identity.get("Arn", "")
+    user_id = identity.get("UserId", "")
+
+    print(f"Caller account: {account}")
+    print(f"Caller ARN: {arn}")
+    print(f"Caller UserId: {user_id}")
+
+    if ":assumed-role/" in arn:
+        principal_kind = "assumed IAM role (typical for SSO or role chaining)"
+    elif ":user/" in arn:
+        principal_kind = "IAM user (typical for static access keys in credentials file)"
+    elif ":federated-user/" in arn:
+        principal_kind = "federated user"
+    else:
+        principal_kind = "other IAM principal"
+
+    print(f"Principal kind: {principal_kind}")
+    credential_summary.update(
+        {
+            "account": account,
+            "arn": arn,
+            "user_id": user_id,
+            "principal_kind": principal_kind,
+        }
+    )
+
+    if expected_account_id and account and account != str(expected_account_id):
+        print(
+            "WARNING: Caller account does not match configured AWS_ACCOUNT_ID. "
+            "CDK will target the configured account but act as this identity — "
+            "deployments and lookups may fail. Set AWS_PROFILE to your SSO profile "
+            "and unset AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY if needed."
+        )
+        credential_summary["account_mismatch"] = True
+    elif expected_account_id and account == str(expected_account_id):
+        print("Caller account matches configured AWS_ACCOUNT_ID.")
+
+    if profile == "(not set — using default profile chain)":
+        print(
+            "TIP: Set AWS_PROFILE to your SSO profile name so Python and the CDK CLI "
+            "(Node) use the same session. Example: "
+            '$env:AWS_PROFILE = "YourSsoProfileName"'
+        )
+
+    print("--- End AWS credential context ---\n")
+    return credential_summary
+
+
+# --- Function to load context from file ---
+def load_context_from_file(app: App, file_path: str):
+    if os.path.exists(file_path):
+        with open(file_path, "r", encoding="utf-8") as f:
+            context_data = json.load(f)
+            for key, value in context_data.items():
+                app.node.set_context(key, value)
+            print(f"Loaded context from {file_path}")
+    else:
+        print(f"Context file not found: {file_path}")
+
+
+# --- Helper to parse environment variables into lists ---
+def _get_env_list(env_var_name: str) -> List[str]:
+    """Parses a comma-separated environment variable into a list of strings."""
+    value = env_var_name[1:-1].strip().replace('"', "").replace("'", "")
+    if not value:
+        return []
+    # Split by comma and filter out any empty strings that might result from extra commas
+    return [s.strip() for s in value.split(",") if s.strip()]
+
+
+# 1. Try to load CIDR/AZs from environment variables
+if PUBLIC_SUBNETS_TO_USE:
+    PUBLIC_SUBNETS_TO_USE = _get_env_list(PUBLIC_SUBNETS_TO_USE)
+if PRIVATE_SUBNETS_TO_USE:
+    PRIVATE_SUBNETS_TO_USE = _get_env_list(PRIVATE_SUBNETS_TO_USE)
+
+if PUBLIC_SUBNET_CIDR_BLOCKS:
+    PUBLIC_SUBNET_CIDR_BLOCKS = _get_env_list("PUBLIC_SUBNET_CIDR_BLOCKS")
+if PUBLIC_SUBNET_AVAILABILITY_ZONES:
+    PUBLIC_SUBNET_AVAILABILITY_ZONES = _get_env_list("PUBLIC_SUBNET_AVAILABILITY_ZONES")
+if PRIVATE_SUBNET_CIDR_BLOCKS:
+    PRIVATE_SUBNET_CIDR_BLOCKS = _get_env_list("PRIVATE_SUBNET_CIDR_BLOCKS")
+if PRIVATE_SUBNET_AVAILABILITY_ZONES:
+    PRIVATE_SUBNET_AVAILABILITY_ZONES = _get_env_list(
+        "PRIVATE_SUBNET_AVAILABILITY_ZONES"
+    )
+
+if POLICY_FILE_LOCATIONS:
+    POLICY_FILE_LOCATIONS = _get_env_list(POLICY_FILE_LOCATIONS)
+
+
+def check_for_existing_role(role_name: str):
+    try:
+        iam = boto3.client("iam")
+        # iam.get_role(RoleName=role_name)
+
+        response = iam.get_role(RoleName=role_name)
+        role = response["Role"]["Arn"]
+
+        print("Response Role:", role)
+
+        return True, role, ""
+    except iam.exceptions.NoSuchEntityException:
+        return False, "", ""
+    except Exception as e:
+        raise Exception("Getting information on IAM role failed due to:", e)
+
+
+from typing import List
+
+# Assume POLICY_FILE_LOCATIONS is defined globally or passed as a default
+# For example:
+# POLICY_FILE_LOCATIONS = ["./policies/my_read_policy.json", "./policies/my_write_policy.json"]
+
+
+def add_statement_to_policy(role: iam.IRole, policy_document: Dict[str, Any]):
+    """
+    Adds individual policy statements from a parsed policy document to a CDK Role.
+
+    Args:
+        role: The CDK Role construct to attach policies to.
+        policy_document: A Python dictionary representing an IAM policy document.
+    """
+    # Ensure the loaded JSON is a valid policy document structure
+    if "Statement" not in policy_document or not isinstance(
+        policy_document["Statement"], list
+    ):
+        print("Warning: Policy document does not contain a 'Statement' list. Skipping.")
+        return  # Do not return role, just log and exit
+
+    for statement_dict in policy_document["Statement"]:
+        try:
+            # Create a CDK PolicyStatement from the dictionary
+            cdk_policy_statement = iam.PolicyStatement.from_json(statement_dict)
+
+            # Add the policy statement to the role
+            role.add_to_policy(cdk_policy_statement)
+            print(f"  - Added statement: {statement_dict.get('Sid', 'No Sid')}")
+        except Exception as e:
+            print(
+                f"Warning: Could not process policy statement: {statement_dict}. Error: {e}"
+            )
+
+
+def add_custom_policies(
+    scope: Construct,  # Not strictly used here, but good practice if you expand to ManagedPolicies
+    role: iam.IRole,
+    policy_file_locations: Optional[List[str]] = None,
+    custom_policy_text: Optional[str] = None,
+) -> iam.IRole:
+    """
+    Loads custom policies from JSON files or a string and attaches them to a CDK Role.
+
+    Args:
+        scope: The scope in which to define constructs (if needed, e.g., for iam.ManagedPolicy).
+        role: The CDK Role construct to attach policies to.
+        policy_file_locations: List of file paths to JSON policy documents.
+        custom_policy_text: A JSON string representing a policy document.
+
+    Returns:
+        The modified CDK Role construct.
+    """
+    if policy_file_locations is None:
+        policy_file_locations = []
+
+    current_source = "unknown source"  # For error messages
+
+    try:
+        if policy_file_locations:
+            print(f"Attempting to add policies from files to role {role.node.id}...")
+            for path in policy_file_locations:
+                current_source = f"file: {path}"
+                try:
+                    with open(path, "r") as f:
+                        policy_document = json.load(f)
+                    print(f"Processing policy from {current_source}...")
+                    add_statement_to_policy(role, policy_document)
+                except FileNotFoundError:
+                    print(f"Warning: Policy file not found at {path}. Skipping.")
+                except json.JSONDecodeError as e:
+                    print(
+                        f"Warning: Invalid JSON in policy file {path}: {e}. Skipping."
+                    )
+                except Exception as e:
+                    print(
+                        f"An unexpected error occurred processing policy from {path}: {e}. Skipping."
+                    )
+
+        if custom_policy_text:
+            current_source = "custom policy text string"
+            print(
+                f"Attempting to add policy from custom text to role {role.node.id}..."
+            )
+            try:
+                # *** FIX: Parse the JSON string into a Python dictionary ***
+                policy_document = json.loads(custom_policy_text)
+                print(f"Processing policy from {current_source}...")
+                add_statement_to_policy(role, policy_document)
+            except json.JSONDecodeError as e:
+                print(f"Warning: Invalid JSON in custom_policy_text: {e}. Skipping.")
+            except Exception as e:
+                print(
+                    f"An unexpected error occurred processing policy from custom_policy_text: {e}. Skipping."
+                )
+
+        # You might want a final success message, but individual processing messages are also good.
+        print(f"Finished processing custom policies for role {role.node.id}.")
+
+    except Exception as e:
+        print(
+            f"An unhandled error occurred during policy addition for {current_source}: {e}"
+        )
+
+    return role
+
+
+# Import the S3 Bucket class if you intend to return a CDK object later
+# from aws_cdk import aws_s3 as s3
+
+
+def check_s3_bucket_exists(
+    bucket_name: str,
+):  # Return type hint depends on what you return
+    """
+    Checks if an S3 bucket with the given name exists and is accessible.
+
+    Args:
+        bucket_name: The name of the S3 bucket to check.
+
+    Returns:
+        A tuple: (bool indicating existence, optional S3 Bucket object or None)
+        Note: Returning a Boto3 S3 Bucket object from here is NOT ideal
+              for direct use in CDK. You'll likely only need the boolean result
+              or the bucket name for CDK lookups/creations.
+              For this example, let's return the boolean and the name.
+    """
+    s3_client = boto3.client("s3")
+    try:
+        # Use head_bucket to check for existence and access
+        s3_client.head_bucket(Bucket=bucket_name)
+        print(f"Bucket '{bucket_name}' exists and is accessible.")
+        return True, bucket_name  # Return True and the bucket name
+
+    except ClientError as e:
+        # If a ClientError occurs, check the error code.
+        # '404' means the bucket does not exist.
+        # '403' means the bucket exists but you don't have permission.
+        error_code = e.response["Error"]["Code"]
+        if error_code == "404":
+            print(f"Bucket '{bucket_name}' does not exist.")
+            return False, None
+        elif error_code == "403":
+            # The bucket exists, but you can't access it.
+            # Depending on your requirements, this might be treated as "exists"
+            # or "not accessible for our purpose". For checking existence,
+            # we'll say it exists here, but note the permission issue.
+            # NOTE - when I tested this, it was returning 403 even for buckets that don't exist. So I will return False instead
+            print(
+                f"Bucket '{bucket_name}' returned 403, which indicates it may exist but is not accessible due to permissions, or that it doesn't exist. Returning False for existence just in case."
+            )
+            return False, bucket_name  # It exists, even if not accessible
+        else:
+            # For other errors, it's better to raise the exception
+            # to indicate something unexpected happened.
+            print(
+                f"An unexpected AWS ClientError occurred checking bucket '{bucket_name}': {e}"
+            )
+            # Decide how to handle other errors - raising might be safer
+            raise  # Re-raise the original exception
+    except Exception as e:
+        print(
+            f"An unexpected non-ClientError occurred checking bucket '{bucket_name}': {e}"
+        )
+        # Decide how to handle other errors
+        raise  # Re-raise the original exception
+
+
+# Example usage in your check_resources.py:
+# exists, bucket_name_if_exists = check_s3_bucket_exists(log_bucket_name)
+# context_data[f"exists:{log_bucket_name}"] = exists
+# # You don't necessarily need to store the name in context if using from_bucket_name
+
+
+# Delete an S3 bucket
+def delete_s3_bucket(bucket_name: str):
+    s3 = boto3.client("s3")
+
+    try:
+        # List and delete all objects
+        response = s3.list_object_versions(Bucket=bucket_name)
+        versions = response.get("Versions", []) + response.get("DeleteMarkers", [])
+        for version in versions:
+            s3.delete_object(
+                Bucket=bucket_name, Key=version["Key"], VersionId=version["VersionId"]
+            )
+
+        # Delete the bucket
+        s3.delete_bucket(Bucket=bucket_name)
+        return {"Status": "SUCCESS"}
+    except Exception as e:
+        return {"Status": "FAILED", "Reason": str(e)}
+
+
+# Function to get subnet ID from subnet name
+def get_subnet_id(vpc: str, ec2_client: str, subnet_name: str):
+    response = ec2_client.describe_subnets(
+        Filters=[{"Name": "vpc-id", "Values": [vpc.vpc_id]}]
+    )
+
+    for subnet in response["Subnets"]:
+        if subnet["Tags"] and any(
+            tag["Key"] == "Name" and tag["Value"] == subnet_name
+            for tag in subnet["Tags"]
+        ):
+            return subnet["SubnetId"]
+
+    return None
+
+
+def check_ecr_repo_exists(repo_name: str) -> tuple[bool, dict]:
+    """
+    Checks if an ECR repository with the given name exists.
+
+    Args:
+        repo_name: The name of the ECR repository to check.
+
+    Returns:
+        True if the repository exists, False otherwise.
+    """
+    ecr_client = boto3.client("ecr")
+    try:
+        print("ecr repo_name to check:", repo_name)
+        response = ecr_client.describe_repositories(repositoryNames=[repo_name])
+        # If describe_repositories succeeds and returns a list of repositories,
+        # and the list is not empty, the repository exists.
+        return len(response["repositories"]) > 0, response["repositories"][0]
+    except ClientError as e:
+        # Check for the specific error code indicating the repository doesn't exist
+        if e.response["Error"]["Code"] == "RepositoryNotFoundException":
+            return False, {}
+        else:
+            # Re-raise other exceptions to handle unexpected errors
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, {}
+
+
+def check_codebuild_project_exists(
+    project_name: str,
+):  # Adjust return type hint as needed
+    """
+    Checks if a CodeBuild project with the given name exists.
+
+    Args:
+        project_name: The name of the CodeBuild project to check.
+
+    Returns:
+        A tuple:
+        - The first element is True if the project exists, False otherwise.
+        - The second element is the project object (dictionary) if found,
+          None otherwise.
+    """
+    codebuild_client = boto3.client("codebuild")
+    try:
+        # Use batch_get_projects with a list containing the single project name
+        response = codebuild_client.batch_get_projects(names=[project_name])
+
+        # The response for batch_get_projects includes 'projects' (found)
+        # and 'projectsNotFound' (not found).
+        if response["projects"]:
+            # If the project is found in the 'projects' list
+            print(f"CodeBuild project '{project_name}' found.")
+            project = response["projects"][0]
+            return (
+                True,
+                project["arn"],
+                project.get("serviceRole"),
+            )
+        elif (
+            response["projectsNotFound"]
+            and project_name in response["projectsNotFound"]
+        ):
+            # If the project name is explicitly in the 'projectsNotFound' list
+            print(f"CodeBuild project '{project_name}' not found.")
+            return False, None, None
+        else:
+            # This case is less expected for a single name lookup,
+            # but could happen if there's an internal issue or the response
+            # structure is slightly different than expected for an error.
+            # It's safer to assume it wasn't found if not in 'projects'.
+            print(
+                f"CodeBuild project '{project_name}' not found (not in 'projects' list)."
+            )
+            return False, None, None
+
+    except ClientError as e:
+        # Catch specific ClientErrors. batch_get_projects might not throw
+        # 'InvalidInputException' for a non-existent project name if the
+        # name format is valid. It typically just lists it in projectsNotFound.
+        # However, other ClientErrors are possible (e.g., permissions).
+        print(
+            f"An AWS ClientError occurred checking CodeBuild project '{project_name}': {e}"
+        )
+        # Decide how to handle other ClientErrors - raising might be safer
+        raise  # Re-raise the original exception
+    except Exception as e:
+        print(
+            f"An unexpected non-ClientError occurred checking CodeBuild project '{project_name}': {e}"
+        )
+        # Decide how to handle other errors
+        raise  # Re-raise the original exception
+
+
+def get_vpc_id_by_name(vpc_name: str) -> Optional[str]:
+    """
+    Finds a VPC ID by its 'Name' tag.
+    """
+    ec2_client = boto3.client("ec2")
+    try:
+        response = ec2_client.describe_vpcs(
+            Filters=[{"Name": "tag:Name", "Values": [vpc_name]}]
+        )
+        if response and response["Vpcs"]:
+            vpc_id = response["Vpcs"][0]["VpcId"]
+            print(f"VPC '{vpc_name}' found with ID: {vpc_id}")
+
+            # In get_vpc_id_by_name, after finding VPC ID:
+
+            # Look for NAT Gateways in this VPC
+            ec2_client = boto3.client("ec2")
+            nat_gateways = []
+            try:
+                response = ec2_client.describe_nat_gateways(
+                    Filters=[
+                        {"Name": "vpc-id", "Values": [vpc_id]},
+                        # Optional: Add a tag filter if you consistently tag your NATs
+                        # {'Name': 'tag:Name', 'Values': [f"{prefix}-nat-gateway"]}
+                    ]
+                )
+                nat_gateways = response.get("NatGateways", [])
+            except Exception as e:
+                print(
+                    f"Warning: Could not describe NAT Gateways in VPC '{vpc_id}': {e}"
+                )
+                # Decide how to handle this error - proceed or raise?
+
+            # Decide how to identify the specific NAT Gateway you want to check for.
+
+            return vpc_id, nat_gateways
+        else:
+            print(f"VPC '{vpc_name}' not found.")
+            return None
+    except Exception as e:
+        print(f"An unexpected error occurred finding VPC '{vpc_name}': {e}")
+        raise
+
+
+# --- Helper to fetch all existing subnets in a VPC once ---
+def _get_existing_subnets_in_vpc(vpc_id: str) -> Dict[str, Any]:
+    """
+    Fetches all subnets in a given VPC.
+    Returns a dictionary with 'by_name' (map of name to subnet data),
+    'by_id' (map of id to subnet data), and 'cidr_networks' (list of ipaddress.IPv4Network).
+    """
+    ec2_client = boto3.client("ec2")
+    existing_subnets_data = {
+        "by_name": {},  # {subnet_name: {'id': 'subnet-id', 'cidr': 'x.x.x.x/x'}}
+        "by_id": {},  # {subnet_id: {'name': 'subnet-name', 'cidr': 'x.x.x.x/x/x'}}
+        "cidr_networks": [],  # List of ipaddress.IPv4Network objects
+    }
+    try:
+        subnet_to_route_table: Dict[str, str] = {}
+        rt_response = ec2_client.describe_route_tables(
+            Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
+        )
+        for route_table in rt_response.get("RouteTables", []):
+            route_table_id = route_table["RouteTableId"]
+            for association in route_table.get("Associations", []):
+                associated_subnet_id = association.get("SubnetId")
+                if associated_subnet_id:
+                    subnet_to_route_table[associated_subnet_id] = route_table_id
+
+        response = ec2_client.describe_subnets(
+            Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
+        )
+        for s in response.get("Subnets", []):
+            subnet_id = s["SubnetId"]
+            cidr_block = s.get("CidrBlock")
+            # Extract 'Name' tag, which is crucial for lookup by name
+            name_tag = next(
+                (tag["Value"] for tag in s.get("Tags", []) if tag["Key"] == "Name"),
+                None,
+            )
+
+            subnet_info = {
+                "id": subnet_id,
+                "cidr": cidr_block,
+                "name": name_tag,
+                "az": s.get("AvailabilityZone"),
+                "route_table_id": subnet_to_route_table.get(subnet_id),
+            }
+
+            if name_tag:
+                existing_subnets_data["by_name"][name_tag] = subnet_info
+            existing_subnets_data["by_id"][subnet_id] = subnet_info
+
+            if cidr_block:
+                try:
+                    existing_subnets_data["cidr_networks"].append(
+                        ipaddress.ip_network(cidr_block, strict=False)
+                    )
+                except ValueError:
+                    print(
+                        f"Warning: Existing subnet {subnet_id} has an invalid CIDR: {cidr_block}. Skipping for overlap check."
+                    )
+
+        print(
+            f"Fetched {len(response.get('Subnets', []))} existing subnets from VPC '{vpc_id}'."
+        )
+    except Exception as e:
+        print(
+            f"Error describing existing subnets in VPC '{vpc_id}': {e}. Cannot perform full validation."
+        )
+        raise  # Re-raise if this essential step fails
+
+    return existing_subnets_data
+
+
+# --- Modified validate_subnet_creation_parameters to take pre-fetched data ---
+def validate_subnet_creation_parameters(
+    vpc_id: str,
+    proposed_subnets_data: List[
+        Dict[str, str]
+    ],  # e.g., [{'name': 'my-public-subnet', 'cidr': '10.0.0.0/24', 'az': 'us-east-1a'}]
+    existing_aws_subnets_data: Dict[
+        str, Any
+    ],  # Pre-fetched data from _get_existing_subnets_in_vpc
+) -> None:
+    """
+    Validates proposed subnet names and CIDR blocks against existing AWS subnets
+    in the specified VPC and against each other.
+    This function uses pre-fetched AWS subnet data.
+
+    Args:
+        vpc_id: The ID of the VPC (for logging/error messages).
+        proposed_subnets_data: A list of dictionaries, where each dict represents
+                               a proposed subnet with 'name', 'cidr', and 'az'.
+        existing_aws_subnets_data: Dictionary containing existing AWS subnet data
+                                   (e.g., from _get_existing_subnets_in_vpc).
+
+    Raises:
+        ValueError: If any proposed subnet name or CIDR block
+                    conflicts with existing AWS resources or other proposed resources.
+    """
+    if not proposed_subnets_data:
+        print("No proposed subnet data provided for validation. Skipping.")
+        return
+
+    print(
+        f"--- Starting pre-synth validation for VPC '{vpc_id}' with proposed subnets ---"
+    )
+
+    print("Existing subnet data:", pd.DataFrame(existing_aws_subnets_data["by_name"]))
+
+    existing_aws_subnet_names = set(existing_aws_subnets_data["by_name"].keys())
+    existing_aws_cidr_networks = existing_aws_subnets_data["cidr_networks"]
+
+    # Sets to track names and list to track networks for internal batch consistency
+    proposed_names_seen: set[str] = set()
+    proposed_cidr_networks_seen: List[ipaddress.IPv4Network] = []
+
+    for i, proposed_subnet in enumerate(proposed_subnets_data):
+        subnet_name = proposed_subnet.get("name")
+        cidr_block_str = proposed_subnet.get("cidr")
+        availability_zone = proposed_subnet.get("az")
+
+        if not all([subnet_name, cidr_block_str, availability_zone]):
+            raise ValueError(
+                f"Proposed subnet at index {i} is incomplete. Requires 'name', 'cidr', and 'az'."
+            )
+
+        # 1. Check for duplicate names within the proposed batch
+        if subnet_name in proposed_names_seen:
+            raise ValueError(
+                f"Proposed subnet name '{subnet_name}' is duplicated within the input list."
+            )
+        proposed_names_seen.add(subnet_name)
+
+        # 2. Check for duplicate names against existing AWS subnets
+        if subnet_name in existing_aws_subnet_names:
+            print(
+                f"Proposed subnet name '{subnet_name}' already exists in VPC '{vpc_id}'."
+            )
+
+        # Parse proposed CIDR
+        try:
+            proposed_net = ipaddress.ip_network(cidr_block_str, strict=False)
+        except ValueError as e:
+            raise ValueError(
+                f"Invalid CIDR format '{cidr_block_str}' for proposed subnet '{subnet_name}': {e}"
+            )
+
+        # 3. Check for overlapping CIDRs within the proposed batch
+        for existing_proposed_net in proposed_cidr_networks_seen:
+            if proposed_net.overlaps(existing_proposed_net):
+                raise ValueError(
+                    f"Proposed CIDR '{cidr_block_str}' for subnet '{subnet_name}' "
+                    f"overlaps with another proposed CIDR '{str(existing_proposed_net)}' "
+                    f"within the same batch."
+                )
+
+        # 4. Check for overlapping CIDRs against existing AWS subnets
+        for existing_aws_net in existing_aws_cidr_networks:
+            if proposed_net.overlaps(existing_aws_net):
+                raise ValueError(
+                    f"Proposed CIDR '{cidr_block_str}' for subnet '{subnet_name}' "
+                    f"overlaps with an existing AWS subnet CIDR '{str(existing_aws_net)}' "
+                    f"in VPC '{vpc_id}'."
+                )
+
+        # If all checks pass for this subnet, add its network to the list for subsequent checks
+        proposed_cidr_networks_seen.append(proposed_net)
+        print(
+            f"Validation successful for proposed subnet '{subnet_name}' with CIDR '{cidr_block_str}'."
+        )
+
+    print(
+        f"--- All proposed subnets passed pre-synth validation checks for VPC '{vpc_id}'. ---"
+    )
+
+
+# --- Modified check_subnet_exists_by_name (Uses pre-fetched data) ---
+def check_subnet_exists_by_name(
+    subnet_name: str, existing_aws_subnets_data: Dict[str, Any]
+) -> Tuple[bool, Optional[str]]:
+    """
+    Checks if a subnet with the given name exists within the pre-fetched data.
+
+    Args:
+        subnet_name: The 'Name' tag value of the subnet to check.
+        existing_aws_subnets_data: Dictionary containing existing AWS subnet data
+                                   (e.g., from _get_existing_subnets_in_vpc).
+
+    Returns:
+        A tuple:
+        - The first element is True if the subnet exists, False otherwise.
+        - The second element is the Subnet ID if found, None otherwise.
+    """
+    subnet_info = existing_aws_subnets_data["by_name"].get(subnet_name)
+    if subnet_info:
+        print(f"Subnet '{subnet_name}' found with ID: {subnet_info['id']}")
+        return True, subnet_info["id"]
+    else:
+        print(f"Subnet '{subnet_name}' not found.")
+        return False, None
+
+
+def create_nat_gateway(
+    scope: Construct,
+    public_subnet_for_nat: ec2.ISubnet,  # Expects a proper ISubnet
+    nat_gateway_name: str,
+    nat_gateway_id_context_key: str,
+) -> str:
+    """
+    Creates a single NAT Gateway in the specified public subnet.
+    It does not handle lookup from context; the calling stack should do that.
+    Returns the CloudFormation Ref of the NAT Gateway ID.
+    """
+    print(
+        f"Defining a new NAT Gateway '{nat_gateway_name}' in subnet '{public_subnet_for_nat.subnet_id}'."
+    )
+
+    # Create an Elastic IP for the NAT Gateway
+    eip = ec2.CfnEIP(
+        scope,
+        NAT_GATEWAY_EIP_NAME,
+        tags=[CfnTag(key="Name", value=NAT_GATEWAY_EIP_NAME)],
+    )
+
+    # Create the NAT Gateway
+    nat_gateway_logical_id = nat_gateway_name.replace("-", "") + "NatGateway"
+    nat_gateway = ec2.CfnNatGateway(
+        scope,
+        nat_gateway_logical_id,
+        subnet_id=public_subnet_for_nat.subnet_id,  # Associate with the public subnet
+        allocation_id=eip.attr_allocation_id,  # Associate with the EIP
+        tags=[CfnTag(key="Name", value=nat_gateway_name)],
+    )
+    # The NAT GW depends on the EIP. The dependency on the subnet is implicit via subnet_id.
+    nat_gateway.add_dependency(eip)
+
+    # *** CRUCIAL: Use CfnOutput to export the ID after deployment ***
+    # This is how you will get the ID to put into cdk.context.json
+    CfnOutput(
+        scope,
+        "SingleNatGatewayIdOutput",
+        value=nat_gateway.ref,
+        description=f"Physical ID of the Single NAT Gateway. Add this to cdk.context.json under the key '{nat_gateway_id_context_key}'.",
+        export_name=f"{scope.stack_name}-NatGatewayId",  # Make export name unique
+    )
+
+    print(
+        f"CDK: Defined new NAT Gateway '{nat_gateway.ref}'. Its physical ID will be available in the stack outputs after deployment."
+    )
+    # Return the tokenised reference for use within this synthesis
+    return nat_gateway.ref
+
+
+def create_subnets(
+    scope: Construct,
+    vpc: ec2.IVpc,
+    prefix: str,
+    subnet_names: List[str],
+    cidr_blocks: List[str],
+    availability_zones: List[str],
+    is_public: bool,
+    internet_gateway_id: Optional[str] = None,
+    single_nat_gateway_id: Optional[str] = None,
+) -> Tuple[List[ec2.CfnSubnet], List[ec2.CfnRouteTable]]:
+    """
+    Creates subnets using L2 constructs but returns the underlying L1 Cfn objects
+    for backward compatibility.
+    """
+    # --- Validations remain the same ---
+    if not (len(subnet_names) == len(cidr_blocks) == len(availability_zones) > 0):
+        raise ValueError(
+            "Subnet names, CIDR blocks, and Availability Zones lists must be non-empty and match in length."
+        )
+    if is_public and not internet_gateway_id:
+        raise ValueError("internet_gateway_id must be provided for public subnets.")
+    if not is_public and not single_nat_gateway_id:
+        raise ValueError(
+            "single_nat_gateway_id must be provided for private subnets when using a single NAT Gateway."
+        )
+
+    # --- We will populate these lists with the L1 objects to return ---
+    created_subnets: List[ec2.CfnSubnet] = []
+    created_route_tables: List[ec2.CfnRouteTable] = []
+
+    subnet_type_tag = "public" if is_public else "private"
+
+    for i, subnet_name in enumerate(subnet_names):
+        logical_id = f"{prefix}{subnet_type_tag.capitalize()}Subnet{i+1}"
+
+        # 1. Create the L2 Subnet (this is the easy part)
+        subnet = ec2.Subnet(
+            scope,
+            logical_id,
+            vpc_id=vpc.vpc_id,
+            cidr_block=cidr_blocks[i],
+            availability_zone=availability_zones[i],
+            map_public_ip_on_launch=is_public,
+        )
+        Tags.of(subnet).add("Name", subnet_name)
+        Tags.of(subnet).add("Type", subnet_type_tag)
+
+        if is_public:
+            # The subnet's route_table is automatically created by the L2 Subnet construct
+            try:
+                subnet.add_route(
+                    "DefaultInternetRoute",  # A logical ID for the CfnRoute resource
+                    router_id=internet_gateway_id,
+                    router_type=ec2.RouterType.GATEWAY,
+                    # destination_cidr_block="0.0.0.0/0" is the default for this method
+                )
+            except Exception as e:
+                print("Could not create IGW route for public subnet due to:", e)
+            print(f"CDK: Defined public L2 subnet '{subnet_name}' and added IGW route.")
+        else:
+            try:
+                # Using .add_route() for private subnets as well for consistency
+                subnet.add_route(
+                    "DefaultNatRoute",  # A logical ID for the CfnRoute resource
+                    router_id=single_nat_gateway_id,
+                    router_type=ec2.RouterType.NAT_GATEWAY,
+                )
+            except Exception as e:
+                print("Could not create NAT gateway route for public subnet due to:", e)
+            print(
+                f"CDK: Defined private L2 subnet '{subnet_name}' and added NAT GW route."
+            )
+
+        route_table = subnet.route_table
+
+        created_subnets.append(subnet)
+        created_route_tables.append(route_table)
+
+    return created_subnets, created_route_tables
+
+
+def ingress_rule_exists(security_group: str, peer: str, port: str):
+    for rule in security_group.connections.security_groups:
+        if port:
+            if rule.peer == peer and rule.connection == port:
+                return True
+        else:
+            if rule.peer == peer:
+                return True
+    return False
+
+
+def check_for_existing_user_pool(user_pool_name: str):
+    cognito_client = boto3.client("cognito-idp")
+    list_pools_response = cognito_client.list_user_pools(
+        MaxResults=60
+    )  # MaxResults up to 60
+
+    # ListUserPools might require pagination if you have more than 60 pools
+    # This simple example doesn't handle pagination, which could miss your pool
+
+    existing_user_pool_id = ""
+
+    for pool in list_pools_response.get("UserPools", []):
+        if pool.get("Name") == user_pool_name:
+            existing_user_pool_id = pool["Id"]
+            print(
+                f"Found existing user pool by name '{user_pool_name}' with ID: {existing_user_pool_id}"
+            )
+            break  # Found the one we're looking for
+
+    if existing_user_pool_id:
+        return True, existing_user_pool_id, pool
+    else:
+        return False, "", ""
+
+
+def check_for_existing_user_pool_client(user_pool_id: str, user_pool_client_name: str):
+    """
+    Checks if a Cognito User Pool Client with the given name exists in the specified User Pool.
+
+    Args:
+        user_pool_id: The ID of the Cognito User Pool.
+        user_pool_client_name: The name of the User Pool Client to check for.
+
+    Returns:
+        A tuple:
+        - True, client_id, client_details if the client exists.
+        - False, "", {} otherwise.
+    """
+    cognito_client = boto3.client("cognito-idp")
+    next_token = "string"
+
+    while True:
+        try:
+            response = cognito_client.list_user_pool_clients(
+                UserPoolId=user_pool_id, MaxResults=60, NextToken=next_token
+            )
+        except cognito_client.exceptions.ResourceNotFoundException:
+            print(f"Error: User pool with ID '{user_pool_id}' not found.")
+            return False, "", {}
+
+        except cognito_client.exceptions.InvalidParameterException:
+            print(f"Error: No app clients for '{user_pool_id}' found.")
+            return False, "", {}
+
+        except Exception as e:
+            print("Could not check User Pool clients due to:", e)
+
+        for client in response.get("UserPoolClients", []):
+            if client.get("ClientName") == user_pool_client_name:
+                print(
+                    f"Found existing user pool client '{user_pool_client_name}' with ID: {client['ClientId']}"
+                )
+                return True, client["ClientId"], client
+
+        next_token = response.get("NextToken")
+        if not next_token:
+            break
+
+    return False, "", {}
+
+
+def check_for_secret(secret_name: str, secret_value: dict = ""):
+    """
+    Checks if a Secrets Manager secret with the given name exists.
+    If it doesn't exist, it creates the secret.
+
+    Args:
+        secret_name: The name of the Secrets Manager secret.
+        secret_value: A dictionary containing the key-value pairs for the secret.
+
+    Returns:
+        True if the secret existed or was created, False otherwise (due to other errors).
+    """
+    secretsmanager_client = boto3.client("secretsmanager")
+
+    try:
+        # Try to get the secret. If it doesn't exist, a ResourceNotFoundException will be raised.
+        secret_value = secretsmanager_client.get_secret_value(SecretId=secret_name)
+        print("Secret already exists.")
+        return True, secret_value
+    except secretsmanager_client.exceptions.ResourceNotFoundException:
+        print("Secret not found")
+        return False, {}
+    except Exception as e:
+        # Handle other potential exceptions during the get operation
+        print(f"Error checking for secret: {e}")
+        return False, {}
+
+
+def check_alb_exists(
+    load_balancer_name: str, region_name: str = None
+) -> tuple[bool, dict]:
+    """
+    Checks if an Application Load Balancer (ALB) with the given name exists.
+
+    Args:
+        load_balancer_name: The name of the ALB to check.
+        region_name: The AWS region to check in.  If None, uses the default
+                     session region.
+
+    Returns:
+        A tuple:
+        - The first element is True if the ALB exists, False otherwise.
+        - The second element is the ALB object (dictionary) if found,
+          None otherwise.  Specifically, it returns the first element of
+          the LoadBalancers list from the describe_load_balancers response.
+    """
+    if region_name:
+        elbv2_client = boto3.client("elbv2", region_name=region_name)
+    else:
+        elbv2_client = boto3.client("elbv2")
+    try:
+        response = elbv2_client.describe_load_balancers(Names=[load_balancer_name])
+        if response["LoadBalancers"]:
+            return (
+                True,
+                response["LoadBalancers"][0],
+            )  # Return True and the first ALB object
+        else:
+            return False, {}
+    except ClientError as e:
+        #  If the error indicates the ALB doesn't exist, return False
+        if e.response["Error"]["Code"] == "LoadBalancerNotFound":
+            return False, {}
+        else:
+            # Re-raise other exceptions
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, {}
+
+
+def check_fargate_task_definition_exists(
+    task_definition_name: str, region_name: str = None
+) -> tuple[bool, dict]:
+    """
+    Checks if a Fargate task definition with the given name exists.
+
+    Args:
+        task_definition_name: The name or ARN of the task definition to check.
+        region_name: The AWS region to check in. If None, uses the default
+                     session region.
+
+    Returns:
+        A tuple:
+        - The first element is True if the task definition exists, False otherwise.
+        - The second element is the task definition object (dictionary) if found,
+          None otherwise.  Specifically, it returns the first element of the
+          taskDefinitions list from the describe_task_definition response.
+    """
+    if region_name:
+        ecs_client = boto3.client("ecs", region_name=region_name)
+    else:
+        ecs_client = boto3.client("ecs")
+    try:
+        response = ecs_client.describe_task_definition(
+            taskDefinition=task_definition_name
+        )
+        # If describe_task_definition succeeds, it returns the task definition.
+        # We can directly return True and the task definition.
+        return True, response["taskDefinition"]
+    except ClientError as e:
+        # Check for the error code indicating the task definition doesn't exist.
+        if (
+            e.response["Error"]["Code"] == "ClientException"
+            and "Task definition" in e.response["Message"]
+            and "does not exist" in e.response["Message"]
+        ):
+            return False, {}
+        else:
+            # Re-raise other exceptions.
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, {}
+
+
+def check_ecs_service_exists(
+    cluster_name: str, service_name: str, region_name: str = None
+) -> tuple[bool, dict]:
+    """
+    Checks if an ECS service with the given name exists in the specified cluster.
+
+    Args:
+        cluster_name: The name or ARN of the ECS cluster.
+        service_name: The name of the ECS service to check.
+        region_name: The AWS region to check in. If None, uses the default
+                     session region.
+
+    Returns:
+        A tuple:
+        - The first element is True if the service exists, False otherwise.
+        - The second element is the service object (dictionary) if found,
+          None otherwise.
+    """
+    if region_name:
+        ecs_client = boto3.client("ecs", region_name=region_name)
+    else:
+        ecs_client = boto3.client("ecs")
+    try:
+        response = ecs_client.describe_services(
+            cluster=cluster_name, services=[service_name]
+        )
+        if response["services"]:
+            return (
+                True,
+                response["services"][0],
+            )  # Return True and the first service object
+        else:
+            return False, {}
+    except ClientError as e:
+        # Check for the error code indicating the service doesn't exist.
+        if e.response["Error"]["Code"] == "ClusterNotFoundException":
+            return False, {}
+        elif e.response["Error"]["Code"] == "ServiceNotFoundException":
+            return False, {}
+        else:
+            # Re-raise other exceptions.
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, {}
+
+
+def check_cloudfront_distribution_exists(
+    distribution_name: str, region_name: str = None
+) -> tuple[bool, dict | None]:
+    """
+    Checks if a CloudFront distribution with the given name exists.
+
+    Args:
+        distribution_name: The name of the CloudFront distribution to check.
+        region_name: The AWS region to check in. If None, uses the default
+                     session region.  Note: CloudFront is a global service,
+                     so the region is usually 'us-east-1', but this parameter
+                     is included for completeness.
+
+    Returns:
+        A tuple:
+        - The first element is True if the distribution exists, False otherwise.
+        - The second element is the distribution object (dictionary) if found,
+          None otherwise.  Specifically, it returns the first element of the
+          DistributionList from the ListDistributions response.
+    """
+    if region_name:
+        cf_client = boto3.client("cloudfront", region_name=region_name)
+    else:
+        cf_client = boto3.client("cloudfront")
+    try:
+        response = cf_client.list_distributions()
+        if "Items" in response["DistributionList"]:
+            for distribution in response["DistributionList"]["Items"]:
+                # CloudFront doesn't directly filter by name, so we have to iterate.
+                if (
+                    distribution["AliasSet"]["Items"]
+                    and distribution["AliasSet"]["Items"][0] == distribution_name
+                ):
+                    return True, distribution
+            return False, None
+        else:
+            return False, None
+    except ClientError as e:
+        #  If the error indicates the Distribution doesn't exist, return False
+        if e.response["Error"]["Code"] == "NoSuchDistribution":
+            return False, None
+        else:
+            # Re-raise other exceptions
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, None
+
+
+def create_web_acl_with_common_rules(
+    scope: Construct, web_acl_name: str, waf_scope: str = "CLOUDFRONT"
+):
+    """
+    Use CDK to create a web ACL based on an AWS common rule set with overrides.
+    This function now expects a 'scope' argument, typically 'self' from your stack,
+    as CfnWebACL requires a construct scope.
+    """
+
+    # Create full list of rules
+    rules = []
+    aws_ruleset_names = [
+        "AWSManagedRulesCommonRuleSet",
+        "AWSManagedRulesKnownBadInputsRuleSet",
+        "AWSManagedRulesAmazonIpReputationList",
+    ]
+
+    # Use a separate counter to assign unique priorities sequentially
+    priority_counter = 1
+
+    for aws_rule_name in aws_ruleset_names:
+        current_rule_action_overrides = None
+
+        # All managed rule groups need an override_action.
+        # 'none' means use the managed rule group's default action.
+        current_override_action = wafv2.CfnWebACL.OverrideActionProperty(none={})
+
+        current_priority = priority_counter
+        priority_counter += 1
+
+        if aws_rule_name == "AWSManagedRulesCommonRuleSet":
+            current_rule_action_overrides = [
+                wafv2.CfnWebACL.RuleActionOverrideProperty(
+                    name="SizeRestrictions_BODY",
+                    action_to_use=wafv2.CfnWebACL.RuleActionProperty(allow={}),
+                )
+            ]
+            # No need to set current_override_action here, it's already set above.
+            # If you wanted this specific rule to have a *fixed* priority, you'd handle it differently
+            # For now, it will get priority 1 from the counter.
+
+        rule_property = wafv2.CfnWebACL.RuleProperty(
+            name=aws_rule_name,
+            priority=current_priority,
+            statement=wafv2.CfnWebACL.StatementProperty(
+                managed_rule_group_statement=wafv2.CfnWebACL.ManagedRuleGroupStatementProperty(
+                    vendor_name="AWS",
+                    name=aws_rule_name,
+                    rule_action_overrides=current_rule_action_overrides,
+                )
+            ),
+            visibility_config=wafv2.CfnWebACL.VisibilityConfigProperty(
+                cloud_watch_metrics_enabled=True,
+                metric_name=aws_rule_name,
+                sampled_requests_enabled=True,
+            ),
+            override_action=current_override_action,  # THIS IS THE CRUCIAL PART FOR ALL MANAGED RULES
+        )
+
+        rules.append(rule_property)
+
+    # Add the rate limit rule
+    rate_limit_priority = priority_counter  # Use the next available priority
+    rules.append(
+        wafv2.CfnWebACL.RuleProperty(
+            name="RateLimitRule",
+            priority=rate_limit_priority,
+            statement=wafv2.CfnWebACL.StatementProperty(
+                rate_based_statement=wafv2.CfnWebACL.RateBasedStatementProperty(
+                    limit=1000, aggregate_key_type="IP"
+                )
+            ),
+            visibility_config=wafv2.CfnWebACL.VisibilityConfigProperty(
+                cloud_watch_metrics_enabled=True,
+                metric_name="RateLimitRule",
+                sampled_requests_enabled=True,
+            ),
+            action=wafv2.CfnWebACL.RuleActionProperty(block={}),
+        )
+    )
+
+    web_acl = wafv2.CfnWebACL(
+        scope,
+        "WebACL",
+        name=web_acl_name,
+        default_action=wafv2.CfnWebACL.DefaultActionProperty(allow={}),
+        scope=waf_scope,
+        visibility_config=wafv2.CfnWebACL.VisibilityConfigProperty(
+            cloud_watch_metrics_enabled=True,
+            metric_name="webACL",
+            sampled_requests_enabled=True,
+        ),
+        rules=rules,
+    )
+
+    CfnOutput(scope, "WebACLArn", value=web_acl.attr_arn)
+
+    return web_acl
+
+
+def check_web_acl_exists(
+    web_acl_name: str, scope: str, region_name: str = None
+) -> tuple[bool, dict]:
+    """
+    Checks if a Web ACL with the given name and scope exists.
+
+    Args:
+        web_acl_name: The name of the Web ACL to check.
+        scope: The scope of the Web ACL ('CLOUDFRONT' or 'REGIONAL').
+        region_name: The AWS region to check in. Required for REGIONAL scope.
+                     If None, uses the default session region.  For CLOUDFRONT,
+                     the region should be 'us-east-1'.
+
+    Returns:
+        A tuple:
+        - The first element is True if the Web ACL exists, False otherwise.
+        - The second element is the Web ACL object (dictionary) if found,
+          None otherwise.
+    """
+    if scope not in ["CLOUDFRONT", "REGIONAL"]:
+        raise ValueError("Scope must be either 'CLOUDFRONT' or 'REGIONAL'")
+
+    if scope == "REGIONAL" and not region_name:
+        raise ValueError("Region name is required for REGIONAL scope")
+
+    if scope == "CLOUDFRONT":
+        region_name = "us-east-1"  # CloudFront scope requires us-east-1
+
+    if region_name:
+        waf_client = boto3.client("wafv2", region_name=region_name)
+    else:
+        waf_client = boto3.client("wafv2")
+    try:
+        response = waf_client.list_web_acls(Scope=scope)
+        if "WebACLs" in response:
+            for web_acl in response["WebACLs"]:
+                if web_acl["Name"] == web_acl_name:
+                    # Describe the Web ACL to get the full object.
+                    describe_response = waf_client.describe_web_acl(
+                        Name=web_acl_name, Scope=scope
+                    )
+                    return True, describe_response["WebACL"]
+            return False, {}
+        else:
+            return False, {}
+    except ClientError as e:
+        # Check for the error code indicating the web ACL doesn't exist.
+        if e.response["Error"]["Code"] == "ResourceNotFoundException":
+            return False, {}
+        else:
+            # Re-raise other exceptions.
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, {}
+
+
+def add_alb_https_listener_with_cert(
+    scope: Construct,
+    logical_id: str,  # A unique ID for this listener construct
+    alb: elb.ApplicationLoadBalancer,
+    acm_certificate_arn: Optional[
+        str
+    ],  # Optional: If None, no HTTPS listener will be created
+    default_target_group: elb.ITargetGroup,  # Mandatory: The target group to forward traffic to
+    listener_port_https: int = 443,
+    listener_open_to_internet: bool = False,  # Be cautious with True, ensure ALB security group restricts access
+    # --- Cognito Authentication Parameters ---
+    enable_cognito_auth: bool = False,
+    cognito_user_pool: Optional[cognito.IUserPool] = None,
+    cognito_user_pool_client: Optional[cognito.IUserPoolClient] = None,
+    cognito_user_pool_domain: Optional[
+        str
+    ] = None,  # E.g., "my-app-domain" for "my-app-domain.auth.region.amazoncognito.com"
+    cognito_auth_scope: Optional[
+        str
+    ] = "openid profile email",  # Default recommended scope
+    cognito_auth_on_unauthenticated_request: elb.UnauthenticatedAction = elb.UnauthenticatedAction.AUTHENTICATE,
+    stickiness_cookie_duration=None,
+    # --- End Cognito Parameters ---
+) -> Optional[elb.ApplicationListener]:
+    """
+    Conditionally adds an HTTPS listener to an ALB with an ACM certificate,
+    and optionally enables Cognito User Pool authentication.
+
+    Args:
+        scope (Construct): The scope in which to define this construct (e.g., your CDK Stack).
+        logical_id (str): A unique logical ID for the listener construct within the stack.
+        alb (elb.ApplicationLoadBalancer): The Application Load Balancer to add the listener to.
+        acm_certificate_arn (Optional[str]): The ARN of the ACM certificate to attach.
+                                             If None, the HTTPS listener will NOT be created.
+        default_target_group (elb.ITargetGroup): The default target group for the listener to forward traffic to.
+                                                 This is mandatory for a functional listener.
+        listener_port_https (int): The HTTPS port to listen on (default: 443).
+        listener_open_to_internet (bool): Whether the listener should allow connections from all sources.
+                                          If False (recommended), ensure your ALB's security group allows
+                                          inbound traffic on this port from desired sources.
+        enable_cognito_auth (bool): Set to True to enable Cognito User Pool authentication.
+        cognito_user_pool (Optional[cognito.IUserPool]): The Cognito User Pool object. Required if enable_cognito_auth is True.
+        cognito_user_pool_client (Optional[cognito.IUserPoolClient]): The Cognito User Pool App Client object. Required if enable_cognito_auth is True.
+        cognito_user_pool_domain (Optional[str]): The domain prefix for your Cognito User Pool. Required if enable_cognito_auth is True.
+        cognito_auth_scope (Optional[str]): The scope for the Cognito authentication.
+        cognito_auth_on_unauthenticated_request (elb.UnauthenticatedAction): Action for unauthenticated requests.
+                                                                           Defaults to AUTHENTICATE (redirect to login).
+
+    Returns:
+        Optional[elb.ApplicationListener]: The created ApplicationListener if successful,
+                                           None if no ACM certificate ARN was provided.
+    """
+    https_listener = None
+    if acm_certificate_arn:
+        certificates_list = [elb.ListenerCertificate.from_arn(acm_certificate_arn)]
+        print(
+            f"Attempting to add ALB HTTPS listener on port {listener_port_https} with ACM certificate: {acm_certificate_arn}"
+        )
+
+        # Determine the default action based on whether Cognito auth is enabled
+        default_action = None
+        if enable_cognito_auth is True:
+            if not all(
+                [cognito_user_pool, cognito_user_pool_client, cognito_user_pool_domain]
+            ):
+                raise ValueError(
+                    "Cognito User Pool, Client, and Domain must be provided if enable_cognito_auth is True."
+                )
+            print(
+                f"Enabling Cognito authentication with User Pool: {cognito_user_pool.user_pool_id}"
+            )
+
+            default_action = elb_act.AuthenticateCognitoAction(
+                next=elb.ListenerAction.forward(
+                    [default_target_group]
+                ),  # After successful auth, forward to TG
+                user_pool=cognito_user_pool,
+                user_pool_client=cognito_user_pool_client,
+                user_pool_domain=cognito_user_pool_domain,
+                scope=cognito_auth_scope,
+                on_unauthenticated_request=cognito_auth_on_unauthenticated_request,
+                session_timeout=stickiness_cookie_duration,
+                # Additional options you might want to configure:
+                # session_cookie_name="AWSELBCookies"
+            )
+        else:
+            default_action = elb.ListenerAction.forward([default_target_group])
+            print("Cognito authentication is NOT enabled for this listener.")
+
+        # Add the HTTPS listener
+        https_listener = alb.add_listener(
+            logical_id,
+            port=listener_port_https,
+            open=listener_open_to_internet,
+            certificates=certificates_list,
+            default_action=default_action,  # Use the determined default action
+        )
+        print(f"ALB HTTPS listener on port {listener_port_https} defined.")
+    else:
+        print("ACM_CERTIFICATE_ARN is not provided. Skipping HTTPS listener creation.")
+
+    return https_listener
+
+
+def ensure_folder_exists(output_folder: str):
+    """Checks if the specified folder exists, creates it if not."""
+
+    if not os.path.exists(output_folder):
+        # Create the folder if it doesn't exist
+        os.makedirs(output_folder, exist_ok=True)
+        print(f"Created the {output_folder} folder.")
+    else:
+        print(f"The {output_folder} folder already exists.")
+
+
+def create_basic_config_env(
+    out_dir: str = "config",
+    S3_LOG_CONFIG_BUCKET_NAME=S3_LOG_CONFIG_BUCKET_NAME,
+    S3_OUTPUT_BUCKET_NAME=S3_OUTPUT_BUCKET_NAME,
+    ACCESS_LOG_DYNAMODB_TABLE_NAME=ACCESS_LOG_DYNAMODB_TABLE_NAME,
+    FEEDBACK_LOG_DYNAMODB_TABLE_NAME=FEEDBACK_LOG_DYNAMODB_TABLE_NAME,
+    USAGE_LOG_DYNAMODB_TABLE_NAME=USAGE_LOG_DYNAMODB_TABLE_NAME,
+):
+    """
+    Create a basic config.env file for the user to use with their newly deployed redaction app.
+    """
+    variables = {
+        "COGNITO_AUTH": "True",
+        "RUN_AWS_FUNCTIONS": "True",
+        "DISPLAY_FILE_NAMES_IN_LOGS": "False",
+        "SESSION_OUTPUT_FOLDER": "True",
+        "SAVE_LOGS_TO_DYNAMODB": "True",
+        "SHOW_COSTS": "True",
+        "SHOW_WHOLE_DOCUMENT_TEXTRACT_CALL_OPTIONS": "True",
+        "LOAD_PREVIOUS_TEXTRACT_JOBS_S3": "True",
+        "DOCUMENT_REDACTION_BUCKET": S3_LOG_CONFIG_BUCKET_NAME,
+        "TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET": S3_OUTPUT_BUCKET_NAME,
+        "ACCESS_LOG_DYNAMODB_TABLE_NAME": ACCESS_LOG_DYNAMODB_TABLE_NAME,
+        "FEEDBACK_LOG_DYNAMODB_TABLE_NAME": FEEDBACK_LOG_DYNAMODB_TABLE_NAME,
+        "USAGE_LOG_DYNAMODB_TABLE_NAME": USAGE_LOG_DYNAMODB_TABLE_NAME,
+    }
+
+    # Write variables to .env file
+    ensure_folder_exists(out_dir + "/")
+    env_file_path = os.path.abspath(os.path.join(out_dir, "config.env"))
+
+    # It's good practice to ensure the file exists before calling set_key repeatedly.
+    # set_key will create it, but for a loop, it might be cleaner to ensure it's empty/exists once.
+    if not os.path.exists(env_file_path):
+        with open(env_file_path, "w"):
+            pass  # Create empty file
+
+    for key, value in variables.items():
+        set_key(env_file_path, key, str(value), quote_mode="never")
+
+    return variables
+
+
+def start_codebuild_build(PROJECT_NAME: str, AWS_REGION: str = AWS_REGION):
+    """
+    Start an existing Codebuild project build
+    """
+
+    # --- Initialize CodeBuild client ---
+    client = boto3.client("codebuild", region_name=AWS_REGION)
+
+    try:
+        print(f"Attempting to start build for project: {PROJECT_NAME}")
+
+        response = client.start_build(projectName=PROJECT_NAME)
+
+        build_id = response["build"]["id"]
+        print(f"Successfully started build with ID: {build_id}")
+        print(f"Build ARN: {response['build']['arn']}")
+        print("Build URL (approximate - construct based on region and ID):")
+        print(
+            f"https://{AWS_REGION}.console.aws.amazon.com/codesuite/codebuild/projects/{PROJECT_NAME}/build/{build_id.split(':')[-1]}/detail"
+        )
+
+        # You can inspect the full response if needed
+        # print("\nFull response:")
+        # import json
+        # print(json.dumps(response, indent=2))
+
+    except client.exceptions.ResourceNotFoundException:
+        print(f"Error: Project '{PROJECT_NAME}' not found in region '{AWS_REGION}'.")
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+
+
+def upload_file_to_s3(
+    local_file_paths: List[str],
+    s3_key: str,
+    s3_bucket: str,
+    RUN_AWS_FUNCTIONS: str = "1",
+):
+    """
+    Uploads a file from local machine to Amazon S3.
+
+    Args:
+    - local_file_path: Local file path(s) of the file(s) to upload.
+    - s3_key: Key (path) to the file in the S3 bucket.
+    - s3_bucket: Name of the S3 bucket.
+
+    Returns:
+    - Message as variable/printed to console
+    """
+    final_out_message = []
+    final_out_message_str = ""
+
+    if RUN_AWS_FUNCTIONS == "1":
+        try:
+            if s3_bucket and local_file_paths:
+
+                s3_client = boto3.client("s3", region_name=AWS_REGION)
+
+                if isinstance(local_file_paths, str):
+                    local_file_paths = [local_file_paths]
+
+                for file in local_file_paths:
+                    if s3_client:
+                        # print(s3_client)
+                        try:
+                            # Get file name off file path
+                            file_name = os.path.basename(file)
+
+                            s3_key_full = s3_key + file_name
+                            print("S3 key: ", s3_key_full)
+
+                            s3_client.upload_file(file, s3_bucket, s3_key_full)
+                            out_message = (
+                                "File " + file_name + " uploaded successfully!"
+                            )
+                            print(out_message)
+
+                        except Exception as e:
+                            out_message = f"Error uploading file(s): {e}"
+                            print(out_message)
+
+                        final_out_message.append(out_message)
+                        final_out_message_str = "\n".join(final_out_message)
+
+                    else:
+                        final_out_message_str = "Could not connect to AWS."
+            else:
+                final_out_message_str = (
+                    "At least one essential variable is empty, could not upload to S3"
+                )
+        except Exception as e:
+            final_out_message_str = "Could not upload files to S3 due to: " + str(e)
+            print(final_out_message_str)
+    else:
+        final_out_message_str = "App not set to run AWS functions"
+
+    return final_out_message_str
+
+
+# Initialize ECS client
+def start_ecs_task(cluster_name, service_name):
+    ecs_client = boto3.client("ecs")
+
+    try:
+        # Update the service to set the desired count to 1
+        ecs_client.update_service(
+            cluster=cluster_name, service=service_name, desiredCount=1
+        )
+        return {
+            "statusCode": 200,
+            "body": f"Service {service_name} in cluster {cluster_name} has been updated to 1 task.",
+        }
+    except Exception as e:
+        return {"statusCode": 500, "body": f"Error updating service: {str(e)}"}

cdk/cdk_stack.py

@@ -0,0 +1,1991 @@
+import json  # You might still need json if loading task_definition.json
+import os
+from typing import Any, Dict, List
+
+from aws_cdk import (
+    CfnOutput,  # <-- Import CfnOutput directly
+    Duration,
+    RemovalPolicy,
+    SecretValue,
+    Stack,
+)
+from aws_cdk import aws_cloudfront as cloudfront
+from aws_cdk import aws_cloudfront_origins as origins
+from aws_cdk import aws_codebuild as codebuild
+from aws_cdk import aws_cognito as cognito
+from aws_cdk import aws_dynamodb as dynamodb  # Import the DynamoDB module
+from aws_cdk import aws_ec2 as ec2
+from aws_cdk import aws_ecr as ecr
+from aws_cdk import aws_ecs as ecs
+from aws_cdk import aws_elasticloadbalancingv2 as elbv2
+from aws_cdk import aws_iam as iam
+from aws_cdk import aws_kms as kms
+from aws_cdk import aws_logs as logs
+from aws_cdk import aws_s3 as s3
+from aws_cdk import aws_secretsmanager as secretsmanager
+from aws_cdk import aws_wafv2 as wafv2
+from cdk_config import (
+    ACCESS_LOG_DYNAMODB_TABLE_NAME,
+    ACM_SSL_CERTIFICATE_ARN,
+    ALB_NAME,
+    ALB_NAME_SECURITY_GROUP_NAME,
+    ALB_TARGET_GROUP_NAME,
+    AWS_ACCOUNT_ID,
+    AWS_MANAGED_TASK_ROLES_LIST,
+    AWS_REGION,
+    CDK_PREFIX,
+    CLOUDFRONT_DISTRIBUTION_NAME,
+    CLOUDFRONT_GEO_RESTRICTION,
+    CLUSTER_NAME,
+    CODEBUILD_PROJECT_NAME,
+    CODEBUILD_ROLE_NAME,
+    COGNITO_ACCESS_TOKEN_VALIDITY,
+    COGNITO_ID_TOKEN_VALIDITY,
+    COGNITO_REDIRECTION_URL,
+    COGNITO_REFRESH_TOKEN_VALIDITY,
+    COGNITO_USER_POOL_CLIENT_NAME,
+    COGNITO_USER_POOL_CLIENT_SECRET_NAME,
+    COGNITO_USER_POOL_DOMAIN_PREFIX,
+    COGNITO_USER_POOL_NAME,
+    CUSTOM_HEADER,
+    CUSTOM_HEADER_VALUE,
+    CUSTOM_KMS_KEY_NAME,
+    DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS,
+    ECR_CDK_REPO_NAME,
+    ECS_LOG_GROUP_NAME,
+    ECS_READ_ONLY_FILE_SYSTEM,
+    ECS_SECURITY_GROUP_NAME,
+    ECS_SERVICE_NAME,
+    ECS_TASK_CPU_SIZE,
+    ECS_TASK_EXECUTION_ROLE_NAME,
+    ECS_TASK_MEMORY_SIZE,
+    ECS_TASK_ROLE_NAME,
+    ECS_USE_FARGATE_SPOT,
+    EXISTING_IGW_ID,
+    EXISTING_LOAD_BALANCER_ARN,
+    EXISTING_LOAD_BALANCER_DNS,
+    FARGATE_TASK_DEFINITION_NAME,
+    FEEDBACK_LOG_DYNAMODB_TABLE_NAME,
+    GITHUB_REPO_BRANCH,
+    GITHUB_REPO_NAME,
+    GITHUB_REPO_USERNAME,
+    GRADIO_SERVER_PORT,
+    LOAD_BALANCER_WEB_ACL_NAME,
+    NAT_GATEWAY_NAME,
+    NEW_VPC_CIDR,
+    NEW_VPC_DEFAULT_NAME,
+    PRIVATE_SUBNET_AVAILABILITY_ZONES,
+    PRIVATE_SUBNET_CIDR_BLOCKS,
+    PRIVATE_SUBNETS_TO_USE,
+    PUBLIC_SUBNET_AVAILABILITY_ZONES,
+    PUBLIC_SUBNET_CIDR_BLOCKS,
+    PUBLIC_SUBNETS_TO_USE,
+    S3_LOG_CONFIG_BUCKET_NAME,
+    S3_OUTPUT_BUCKET_NAME,
+    SAVE_LOGS_TO_DYNAMODB,
+    SINGLE_NAT_GATEWAY_ID,
+    TASK_DEFINITION_FILE_LOCATION,
+    USAGE_LOG_DYNAMODB_TABLE_NAME,
+    USE_CLOUDFRONT,
+    USE_CUSTOM_KMS_KEY,
+    VPC_NAME,
+    WEB_ACL_NAME,
+)
+from cdk_functions import (  # Only keep CDK-native functions
+    add_alb_https_listener_with_cert,
+    add_custom_policies,
+    create_nat_gateway,
+    create_subnets,
+    create_web_acl_with_common_rules,
+)
+from constructs import Construct
+
+
+def _get_env_list(env_var_name: str) -> List[str]:
+    """Parses a comma-separated environment variable into a list of strings."""
+    value = env_var_name[1:-1].strip().replace('"', "").replace("'", "")
+    if not value:
+        return []
+    # Split by comma and filter out any empty strings that might result from extra commas
+    return [s.strip() for s in value.split(",") if s.strip()]
+
+
+# 1. Try to load CIDR/AZs from environment variables
+if PUBLIC_SUBNETS_TO_USE:
+    PUBLIC_SUBNETS_TO_USE = _get_env_list(PUBLIC_SUBNETS_TO_USE)
+if PRIVATE_SUBNETS_TO_USE:
+    PRIVATE_SUBNETS_TO_USE = _get_env_list(PRIVATE_SUBNETS_TO_USE)
+
+if PUBLIC_SUBNET_CIDR_BLOCKS:
+    PUBLIC_SUBNET_CIDR_BLOCKS = _get_env_list("PUBLIC_SUBNET_CIDR_BLOCKS")
+if PUBLIC_SUBNET_AVAILABILITY_ZONES:
+    PUBLIC_SUBNET_AVAILABILITY_ZONES = _get_env_list("PUBLIC_SUBNET_AVAILABILITY_ZONES")
+if PRIVATE_SUBNET_CIDR_BLOCKS:
+    PRIVATE_SUBNET_CIDR_BLOCKS = _get_env_list("PRIVATE_SUBNET_CIDR_BLOCKS")
+if PRIVATE_SUBNET_AVAILABILITY_ZONES:
+    PRIVATE_SUBNET_AVAILABILITY_ZONES = _get_env_list(
+        "PRIVATE_SUBNET_AVAILABILITY_ZONES"
+    )
+
+if AWS_MANAGED_TASK_ROLES_LIST:
+    AWS_MANAGED_TASK_ROLES_LIST = _get_env_list(AWS_MANAGED_TASK_ROLES_LIST)
+
+
+class CdkStack(Stack):
+
+    def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
+        super().__init__(scope, construct_id, **kwargs)
+
+        # --- Helper to get context values ---
+        def get_context_bool(key: str, default: bool = False) -> bool:
+            value = self.node.try_get_context(key)
+            if value is None:
+                return default
+            if isinstance(value, bool):
+                return value
+            if isinstance(value, str):
+                return value.lower() in ("true", "1", "yes")
+            return bool(value)
+
+        def get_context_str(key: str, default: str = None) -> str:
+            return self.node.try_get_context(key) or default
+
+        def get_context_dict(key: str, default: dict = None) -> dict:
+            return self.node.try_get_context(key) or default
+
+        def get_context_list_of_dicts(key: str) -> List[Dict[str, Any]]:
+            ctx_value = self.node.try_get_context(key)
+            if not isinstance(ctx_value, list):
+                print(
+                    f"Warning: Context key '{key}' not found or not a list. Returning empty list."
+                )
+                return []
+            # Optional: Add validation that all items in the list are dicts
+            return ctx_value
+
+        self.template_options.description = "Deployment of the 'doc_redaction' PDF, image, and XLSX/CSV redaction app. Git repo available at: https://github.com/seanpedrick-case/doc_redaction."
+
+        # --- VPC and Subnets (Assuming VPC is always lookup, Subnets are created/returned by create_subnets) ---
+        new_vpc_created = False
+        if VPC_NAME:
+            vpc_id = get_context_str("vpc_id")
+            if not vpc_id:
+                raise ValueError(
+                    f"VPC '{VPC_NAME}' was not resolved during pre-check (missing "
+                    "'vpc_id' in context). Re-run from the cdk/ directory so "
+                    "precheck.context.json is generated."
+                )
+            availability_zones = list(
+                dict.fromkeys(
+                    (PUBLIC_SUBNET_AVAILABILITY_ZONES or [])
+                    + (PRIVATE_SUBNET_AVAILABILITY_ZONES or [])
+                )
+            )
+            if not availability_zones:
+                raise ValueError(
+                    "vpc_id is in context but no subnet availability zones are "
+                    "configured. Set PUBLIC_SUBNET_AVAILABILITY_ZONES and/or "
+                    "PRIVATE_SUBNET_AVAILABILITY_ZONES in cdk_config.env."
+                )
+            vpc = ec2.Vpc.from_vpc_attributes(
+                self,
+                "VPC",
+                vpc_id=vpc_id,
+                availability_zones=availability_zones,
+            )
+            print(f"Using VPC from pre-check context: {vpc_id}")
+
+        elif NEW_VPC_DEFAULT_NAME:
+            new_vpc_created = True
+            print(
+                f"NEW_VPC_DEFAULT_NAME ('{NEW_VPC_DEFAULT_NAME}') is set. Creating a new VPC."
+            )
+
+            # Configuration for the new VPC
+            # You can make these configurable via context as well, e.g.,
+            # new_vpc_cidr = self.node.try_get_context("new_vpc_cidr") or "10.0.0.0/24"
+            # new_vpc_max_azs = self.node.try_get_context("new_vpc_max_azs") or 2 # Use 2 AZs by default for HA
+            # new_vpc_nat_gateways = self.node.try_get_context("new_vpc_nat_gateways") or new_vpc_max_azs # One NAT GW per AZ for HA
+            # or 1 for cost savings if acceptable
+            if not NEW_VPC_CIDR:
+                raise Exception(
+                    "App has been instructed to create a new VPC but not VPC CDR range provided to variable NEW_VPC_CIDR"
+                )
+
+            print("Provided NEW_VPC_CIDR range:", NEW_VPC_CIDR)
+
+            new_vpc_cidr = NEW_VPC_CIDR
+            new_vpc_max_azs = 2  # Creates resources in 2 AZs. Adjust as needed.
+
+            # For "a NAT gateway", you can set nat_gateways=1.
+            # For resilience (NAT GW per AZ), set nat_gateways=new_vpc_max_azs.
+            # The Vpc construct will create NAT Gateway(s) if subnet_type PRIVATE_WITH_EGRESS is used
+            # and nat_gateways > 0.
+            new_vpc_nat_gateways = (
+                1  # Creates a single NAT Gateway for cost-effectiveness.
+            )
+            # If you need one per AZ for higher availability, set this to new_vpc_max_azs.
+
+            vpc = ec2.Vpc(
+                self,
+                "MyNewLogicalVpc",  # This is the CDK construct ID
+                vpc_name=NEW_VPC_DEFAULT_NAME,
+                ip_addresses=ec2.IpAddresses.cidr(new_vpc_cidr),
+                max_azs=new_vpc_max_azs,
+                nat_gateways=new_vpc_nat_gateways,  # Number of NAT gateways to create
+                subnet_configuration=[
+                    ec2.SubnetConfiguration(
+                        name="Public",  # Name prefix for public subnets
+                        subnet_type=ec2.SubnetType.PUBLIC,
+                        cidr_mask=28,  # Adjust CIDR mask as needed (e.g., /24 provides ~250 IPs per subnet)
+                    ),
+                    ec2.SubnetConfiguration(
+                        name="Private",  # Name prefix for private subnets
+                        subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS,  # Ensures these subnets have NAT Gateway access
+                        cidr_mask=28,  # Adjust CIDR mask as needed
+                    ),
+                    # You could also add ec2.SubnetType.PRIVATE_ISOLATED if needed
+                ],
+                # Internet Gateway is created and configured automatically for PUBLIC subnets.
+                # Route tables for public subnets will point to the IGW.
+                # Route tables for PRIVATE_WITH_EGRESS subnets will point to the NAT Gateway(s).
+            )
+            print(
+                f"Successfully created new VPC: {vpc.vpc_id} with name '{NEW_VPC_DEFAULT_NAME}'"
+            )
+            # If nat_gateways > 0, vpc.nat_gateway_ips will contain EIPs if Vpc created them.
+            # vpc.public_subnets, vpc.private_subnets, vpc.isolated_subnets are populated.
+
+        else:
+            raise Exception(
+                "VPC_NAME for current VPC not found, and NEW_VPC_DEFAULT_NAME not found to create a new VPC"
+            )
+
+        # --- Subnet Handling (Check Context and Create/Import) ---
+        # Initialize lists to hold ISubnet objects (L2) and CfnSubnet/CfnRouteTable (L1)
+        # We will store ISubnet for consistency, as CfnSubnet has a .subnet_id property
+        self.public_subnets: List[ec2.ISubnet] = []
+        self.private_subnets: List[ec2.ISubnet] = []
+        # Store L1 CfnRouteTables explicitly if you need to reference them later
+        self.private_route_tables_cfn: List[ec2.CfnRouteTable] = []
+        self.public_route_tables_cfn: List[ec2.CfnRouteTable] = (
+            []
+        )  # New: to store public RTs
+
+        names_to_create_private = []
+        names_to_create_public = []
+
+        if not PUBLIC_SUBNETS_TO_USE and not PRIVATE_SUBNETS_TO_USE:
+            print(
+                "Warning: No public or private subnets specified in *_SUBNETS_TO_USE. Attempting to select from existing VPC subnets."
+            )
+
+            print("vpc.public_subnets:", vpc.public_subnets)
+            print("vpc.private_subnets:", vpc.private_subnets)
+
+            if (
+                vpc.public_subnets
+            ):  # These are already one_per_az if max_azs was used and Vpc created them
+                self.public_subnets.extend(vpc.public_subnets)
+            else:
+                self.node.add_warning("No public subnets found in the VPC.")
+
+            # Get private subnets with egress specifically
+            # selected_private_subnets_with_egress = vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS)
+
+            print(
+                f"Selected from VPC: {len(self.public_subnets)} public, {len(self.private_subnets)} private_with_egress subnets."
+            )
+
+            if (
+                len(self.public_subnets) < 1 or len(self.private_subnets) < 1
+            ):  # Simplified check for new VPC
+                # If new_vpc_max_azs was 1, you'd have 1 of each. If 2, then 2 of each.
+                # The original check ' < 2' might be too strict if new_vpc_max_azs=1
+                pass  # For new VPC, allow single AZ setups if configured that way. The VPC construct ensures one per AZ up to max_azs.
+
+            if not self.public_subnets and not self.private_subnets:
+                print(
+                    "Error: No public or private subnets could be found in the VPC for automatic selection. "
+                    "You must either specify subnets in *_SUBNETS_TO_USE or ensure the VPC has discoverable subnets."
+                )
+                raise RuntimeError("No suitable subnets found for automatic selection.")
+            else:
+                print(
+                    f"Automatically selected {len(self.public_subnets)} public and {len(self.private_subnets)} private subnets based on VPC properties."
+                )
+
+            selected_public_subnets = vpc.select_subnets(
+                subnet_type=ec2.SubnetType.PUBLIC, one_per_az=True
+            )
+            private_subnets_egress = vpc.select_subnets(
+                subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS, one_per_az=True
+            )
+
+            if private_subnets_egress.subnets:
+                self.private_subnets.extend(private_subnets_egress.subnets)
+            else:
+                self.node.add_warning(
+                    "No PRIVATE_WITH_EGRESS subnets found in the VPC."
+                )
+
+            try:
+                private_subnets_isolated = vpc.select_subnets(
+                    subnet_type=ec2.SubnetType.PRIVATE_ISOLATED, one_per_az=True
+                )
+            except Exception as e:
+                private_subnets_isolated = []
+                print("Could not find any isolated subnets due to:", e)
+
+            ###
+            combined_subnet_objects = []
+
+            if private_subnets_isolated:
+                if private_subnets_egress.subnets:
+                    # Add the first PRIVATE_WITH_EGRESS subnet
+                    combined_subnet_objects.append(private_subnets_egress.subnets[0])
+            elif not private_subnets_isolated:
+                if private_subnets_egress.subnets:
+                    # Add the first PRIVATE_WITH_EGRESS subnet
+                    combined_subnet_objects.extend(private_subnets_egress.subnets)
+            else:
+                self.node.add_warning(
+                    "No PRIVATE_WITH_EGRESS subnets found to select the first one."
+                )
+
+            # Add all PRIVATE_ISOLATED subnets *except* the first one (if they exist)
+            try:
+                if len(private_subnets_isolated.subnets) > 1:
+                    combined_subnet_objects.extend(private_subnets_isolated.subnets[1:])
+                elif (
+                    private_subnets_isolated.subnets
+                ):  # Only 1 isolated subnet, add a warning if [1:] was desired
+                    self.node.add_warning(
+                        "Only one PRIVATE_ISOLATED subnet found, private_subnets_isolated.subnets[1:] will be empty."
+                    )
+                else:
+                    self.node.add_warning("No PRIVATE_ISOLATED subnets found.")
+            except Exception as e:
+                print("Could not identify private isolated subnets due to:", e)
+
+            # Create an ec2.SelectedSubnets object from the combined private subnet list.
+            selected_private_subnets = vpc.select_subnets(
+                subnets=combined_subnet_objects
+            )
+
+            print("selected_public_subnets:", selected_public_subnets)
+            print("selected_private_subnets:", selected_private_subnets)
+
+            if (
+                len(selected_public_subnets.subnet_ids) < 2
+                or len(selected_private_subnets.subnet_ids) < 2
+            ):
+                raise Exception(
+                    "Need at least two public or private subnets in different availability zones"
+                )
+
+            if not selected_public_subnets and not selected_private_subnets:
+                # If no subnets could be found even with automatic selection, raise an error.
+                # This ensures the stack doesn't proceed if it absolutely needs subnets.
+                print(
+                    "Error: No existing public or private subnets could be found in the VPC for automatic selection. "
+                    "You must either specify subnets in *_SUBNETS_TO_USE or ensure the VPC has discoverable subnets."
+                )
+                raise RuntimeError("No suitable subnets found for automatic selection.")
+            else:
+                self.public_subnets = selected_public_subnets.subnets
+                self.private_subnets = selected_private_subnets.subnets
+                print(
+                    f"Automatically selected {len(self.public_subnets)} public and {len(self.private_subnets)} private subnets based on VPC discovery."
+                )
+
+                print("self.public_subnets:", self.public_subnets)
+                print("self.private_subnets:", self.private_subnets)
+                # Since subnets are now assigned, we can exit this processing block.
+                # The rest of the original code (which iterates *_SUBNETS_TO_USE) will be skipped.
+
+        checked_public_subnets_ctx = get_context_dict("checked_public_subnets")
+        checked_private_subnets_ctx = get_context_dict("checked_private_subnets")
+
+        public_subnets_data_for_creation_ctx = get_context_list_of_dicts(
+            "public_subnets_to_create"
+        )
+        private_subnets_data_for_creation_ctx = get_context_list_of_dicts(
+            "private_subnets_to_create"
+        )
+
+        # --- 3. Process Public Subnets ---
+        print("\n--- Processing Public Subnets ---")
+        # Import existing public subnets
+        if checked_public_subnets_ctx:
+            for i, subnet_name in enumerate(PUBLIC_SUBNETS_TO_USE):
+                subnet_info = checked_public_subnets_ctx.get(subnet_name)
+                if subnet_info and subnet_info.get("exists"):
+                    subnet_id = subnet_info.get("id")
+                    if not subnet_id:
+                        raise RuntimeError(
+                            f"Context for existing public subnet '{subnet_name}' is missing 'id'."
+                        )
+                    subnet_az = subnet_info.get("az")
+                    if (
+                        not subnet_az
+                        and PUBLIC_SUBNET_AVAILABILITY_ZONES
+                        and i < len(PUBLIC_SUBNET_AVAILABILITY_ZONES)
+                    ):
+                        subnet_az = PUBLIC_SUBNET_AVAILABILITY_ZONES[i]
+                    if not subnet_az:
+                        raise RuntimeError(
+                            f"Context for existing public subnet '{subnet_name}' is missing 'az'."
+                        )
+                    subnet_attrs = {
+                        "subnet_id": subnet_id,
+                        "availability_zone": subnet_az,
+                    }
+                    route_table_id = subnet_info.get("route_table_id")
+                    if route_table_id:
+                        subnet_attrs["route_table_id"] = route_table_id
+                    try:
+                        imported_subnet = ec2.Subnet.from_subnet_attributes(
+                            self,
+                            f"ImportedPublicSubnet{subnet_name.replace('-', '')}{i}",
+                            **subnet_attrs,
+                        )
+                        self.public_subnets.append(imported_subnet)
+                        print(
+                            f"Imported existing public subnet: {subnet_name} (ID: {subnet_id})"
+                        )
+                    except Exception as e:
+                        raise RuntimeError(
+                            f"Failed to import public subnet '{subnet_name}' with ID '{subnet_id}'. Error: {e}"
+                        )
+
+        # Create new public subnets based on public_subnets_data_for_creation_ctx
+        if public_subnets_data_for_creation_ctx:
+            names_to_create_public = [
+                s["name"] for s in public_subnets_data_for_creation_ctx
+            ]
+            cidrs_to_create_public = [
+                s["cidr"] for s in public_subnets_data_for_creation_ctx
+            ]
+            azs_to_create_public = [
+                s["az"] for s in public_subnets_data_for_creation_ctx
+            ]
+
+            if names_to_create_public:
+                print(
+                    f"Attempting to create {len(names_to_create_public)} new public subnets: {names_to_create_public}"
+                )
+                newly_created_public_subnets, newly_created_public_rts_cfn = (
+                    create_subnets(
+                        self,
+                        vpc,
+                        CDK_PREFIX,
+                        names_to_create_public,
+                        cidrs_to_create_public,
+                        azs_to_create_public,
+                        is_public=True,
+                        internet_gateway_id=EXISTING_IGW_ID,
+                    )
+                )
+                self.public_subnets.extend(newly_created_public_subnets)
+                self.public_route_tables_cfn.extend(newly_created_public_rts_cfn)
+
+        if (
+            not self.public_subnets
+            and not names_to_create_public
+            and not PUBLIC_SUBNETS_TO_USE
+        ):
+            raise Exception("No public subnets found or created, exiting.")
+
+        # --- NAT Gateway Creation/Lookup ---
+        print("Creating NAT gateway/located existing")
+        self.single_nat_gateway_id = None
+
+        nat_gw_id_from_context = SINGLE_NAT_GATEWAY_ID or get_context_str(
+            "id:NatGateway"
+        )
+
+        if nat_gw_id_from_context:
+            print(
+                f"Using existing NAT Gateway ID from context: {nat_gw_id_from_context}"
+            )
+            self.single_nat_gateway_id = nat_gw_id_from_context
+
+        elif (
+            new_vpc_created
+            and new_vpc_nat_gateways > 0
+            and hasattr(vpc, "nat_gateways")
+            and vpc.nat_gateways
+        ):
+            self.single_nat_gateway_id = vpc.nat_gateways[0].gateway_id
+            print(
+                f"Using NAT Gateway {self.single_nat_gateway_id} created by the new VPC construct."
+            )
+
+        if not self.single_nat_gateway_id:
+            print("Creating a new NAT gateway")
+
+            if hasattr(vpc, "nat_gateways") and vpc.nat_gateways:
+                print("Existing NAT gateway found in vpc")
+                pass
+
+                # If not in context, create a new one, but only if we have a public subnet.
+            elif self.public_subnets:
+                print("NAT Gateway ID not found in context. Creating a new one.")
+                # Place the NAT GW in the first available public subnet
+                first_public_subnet = self.public_subnets[0]
+
+                self.single_nat_gateway_id = create_nat_gateway(
+                    self,
+                    first_public_subnet,
+                    nat_gateway_name=NAT_GATEWAY_NAME,
+                    nat_gateway_id_context_key=SINGLE_NAT_GATEWAY_ID,
+                )
+            else:
+                print(
+                    "WARNING: No public subnets available and NAT gateway not found in existing VPC. Cannot create a NAT Gateway."
+                )
+
+        # --- 4. Process Private Subnets ---
+        print("\n--- Processing Private Subnets ---")
+        if checked_private_subnets_ctx:
+            for i, subnet_name in enumerate(PRIVATE_SUBNETS_TO_USE):
+                subnet_info = checked_private_subnets_ctx.get(subnet_name)
+                if subnet_info and subnet_info.get("exists"):
+                    subnet_id = subnet_info.get("id")
+                    if not subnet_id:
+                        raise RuntimeError(
+                            f"Context for existing private subnet '{subnet_name}' is missing 'id'."
+                        )
+                    subnet_az = subnet_info.get("az")
+                    if (
+                        not subnet_az
+                        and PRIVATE_SUBNET_AVAILABILITY_ZONES
+                        and i < len(PRIVATE_SUBNET_AVAILABILITY_ZONES)
+                    ):
+                        subnet_az = PRIVATE_SUBNET_AVAILABILITY_ZONES[i]
+                    if not subnet_az:
+                        raise RuntimeError(
+                            f"Context for existing private subnet '{subnet_name}' is missing 'az'."
+                        )
+                    subnet_attrs = {
+                        "subnet_id": subnet_id,
+                        "availability_zone": subnet_az,
+                    }
+                    route_table_id = subnet_info.get("route_table_id")
+                    if route_table_id:
+                        subnet_attrs["route_table_id"] = route_table_id
+                    try:
+                        imported_subnet = ec2.Subnet.from_subnet_attributes(
+                            self,
+                            f"ImportedPrivateSubnet{subnet_name.replace('-', '')}{i}",
+                            **subnet_attrs,
+                        )
+                        self.private_subnets.append(imported_subnet)
+                        print(
+                            f"Imported existing private subnet: {subnet_name} (ID: {subnet_id})"
+                        )
+                    except Exception as e:
+                        raise RuntimeError(
+                            f"Failed to import private subnet '{subnet_name}' with ID '{subnet_id}'. Error: {e}"
+                        )
+
+        # Create new private subnets
+        if private_subnets_data_for_creation_ctx:
+            names_to_create_private = [
+                s["name"] for s in private_subnets_data_for_creation_ctx
+            ]
+            cidrs_to_create_private = [
+                s["cidr"] for s in private_subnets_data_for_creation_ctx
+            ]
+            azs_to_create_private = [
+                s["az"] for s in private_subnets_data_for_creation_ctx
+            ]
+
+            if names_to_create_private:
+                print(
+                    f"Attempting to create {len(names_to_create_private)} new private subnets: {names_to_create_private}"
+                )
+                # --- CALL THE NEW CREATE_SUBNETS FUNCTION FOR PRIVATE ---
+                # Ensure self.single_nat_gateway_id is available before this call
+                if not self.single_nat_gateway_id:
+                    raise ValueError(
+                        "A single NAT Gateway ID is required for private subnets but was not resolved."
+                    )
+
+                newly_created_private_subnets_cfn, newly_created_private_rts_cfn = (
+                    create_subnets(
+                        self,
+                        vpc,
+                        CDK_PREFIX,
+                        names_to_create_private,
+                        cidrs_to_create_private,
+                        azs_to_create_private,
+                        is_public=False,
+                        single_nat_gateway_id=self.single_nat_gateway_id,  # Pass the single NAT Gateway ID
+                    )
+                )
+                self.private_subnets.extend(newly_created_private_subnets_cfn)
+                self.private_route_tables_cfn.extend(newly_created_private_rts_cfn)
+                print(
+                    f"Successfully defined {len(newly_created_private_subnets_cfn)} new private subnets and their route tables for creation."
+                )
+        else:
+            print(
+                "No private subnets specified for creation in context ('private_subnets_to_create')."
+            )
+
+        # if not self.private_subnets:
+        #     raise Exception("No private subnets found or created, exiting.")
+
+        if (
+            not self.private_subnets
+            and not names_to_create_private
+            and not PRIVATE_SUBNETS_TO_USE
+        ):
+            # This condition might need adjustment for new VPCs.
+            raise Exception("No private subnets found or created, exiting.")
+
+        # --- 5. Sanity Check and Output ---
+        # Output the single NAT Gateway ID for verification
+        if self.single_nat_gateway_id:
+            CfnOutput(
+                self,
+                "SingleNatGatewayId",
+                value=self.single_nat_gateway_id,
+                description="ID of the single NAT Gateway resolved or created.",
+            )
+        elif (
+            NEW_VPC_DEFAULT_NAME
+            and (self.node.try_get_context("new_vpc_nat_gateways") or 1) > 0
+        ):
+            print(
+                "INFO: A new VPC was created with NAT Gateway(s). Their routing is handled by the VPC construct. No single_nat_gateway_id was explicitly set for separate output."
+            )
+        else:
+            out_message = "WARNING: No single NAT Gateway was resolved or created explicitly by the script's logic after VPC setup."
+            print(out_message)
+            raise Exception(out_message)
+
+        # --- Outputs for other stacks/regions ---
+        # These are crucial for cross-stack, cross-region referencing
+
+        self.params = dict()
+        self.params["vpc_id"] = vpc.vpc_id
+        self.params["private_subnets"] = self.private_subnets
+        self.params["private_route_tables"] = self.private_route_tables_cfn
+        self.params["public_subnets"] = self.public_subnets
+        self.params["public_route_tables"] = self.public_route_tables_cfn
+
+        private_subnet_selection = ec2.SubnetSelection(subnets=self.private_subnets)
+        public_subnet_selection = ec2.SubnetSelection(subnets=self.public_subnets)
+
+        for sub in private_subnet_selection.subnets:
+            print(
+                "private subnet:",
+                sub.subnet_id,
+                "is in availability zone:",
+                sub.availability_zone,
+            )
+
+        for sub in public_subnet_selection.subnets:
+            print(
+                "public subnet:",
+                sub.subnet_id,
+                "is in availability zone:",
+                sub.availability_zone,
+            )
+
+        print("Private subnet route tables:", self.private_route_tables_cfn)
+
+        # Add the S3 Gateway Endpoint to the VPC
+        if names_to_create_private:
+            try:
+                s3_gateway_endpoint = vpc.add_gateway_endpoint(
+                    "S3GatewayEndpoint",
+                    service=ec2.GatewayVpcEndpointAwsService.S3,
+                    subnets=[private_subnet_selection],
+                )
+            except Exception as e:
+                print("Could not add S3 gateway endpoint to subnets due to:", e)
+
+            # Output some useful information
+            CfnOutput(
+                self,
+                "VpcIdOutput",
+                value=vpc.vpc_id,
+                description="The ID of the VPC where the S3 Gateway Endpoint is deployed.",
+            )
+            CfnOutput(
+                self,
+                "S3GatewayEndpointService",
+                value=s3_gateway_endpoint.vpc_endpoint_id,
+                description="The id for the S3 Gateway Endpoint.",
+            )  # Specify the S3 service
+
+        # --- IAM Roles ---
+        if USE_CUSTOM_KMS_KEY == "1":
+            kms_key = kms.Key(
+                self,
+                "RedactionSharedKmsKey",
+                alias=CUSTOM_KMS_KEY_NAME,
+                removal_policy=RemovalPolicy.DESTROY,
+            )
+
+            custom_sts_kms_policy_dict = {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Sid": "STSCallerIdentity",
+                        "Effect": "Allow",
+                        "Action": ["sts:GetCallerIdentity"],
+                        "Resource": "*",
+                    },
+                    {
+                        "Sid": "KMSAccess",
+                        "Effect": "Allow",
+                        "Action": ["kms:Encrypt", "kms:Decrypt", "kms:GenerateDataKey"],
+                        "Resource": kms_key.key_arn,  # Use key_arn, as it's the full ARN, safer than key_id
+                    },
+                ],
+            }
+        else:
+            kms_key = None
+
+            custom_sts_kms_policy_dict = {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Sid": "STSCallerIdentity",
+                        "Effect": "Allow",
+                        "Action": ["sts:GetCallerIdentity"],
+                        "Resource": "*",
+                    },
+                    {
+                        "Sid": "KMSSecretsManagerDecrypt",  # Explicitly add decrypt for default key
+                        "Effect": "Allow",
+                        "Action": ["kms:Decrypt"],
+                        "Resource": f"arn:aws:kms:{AWS_REGION}:{AWS_ACCOUNT_ID}:key/aws/secretsmanager",
+                    },
+                ],
+            }
+        custom_sts_kms_policy = json.dumps(custom_sts_kms_policy_dict, indent=4)
+
+        try:
+            codebuild_role_name = CODEBUILD_ROLE_NAME
+
+            if get_context_bool(f"exists:{codebuild_role_name}"):
+                # If exists, lookup/import the role using ARN from context
+                role_arn = get_context_str(f"arn:{codebuild_role_name}")
+                if not role_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{codebuild_role_name}' is required if role exists."
+                    )
+                codebuild_role = iam.Role.from_role_arn(
+                    self, "CodeBuildRole", role_arn=role_arn
+                )
+                print("Using existing CodeBuild role")
+            else:
+                # If not exists, create the role
+                codebuild_role = iam.Role(
+                    self,
+                    "CodeBuildRole",  # Logical ID
+                    role_name=codebuild_role_name,  # Explicit resource name
+                    assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"),
+                )
+                codebuild_role.add_managed_policy(
+                    iam.ManagedPolicy.from_aws_managed_policy_name(
+                        "EC2InstanceProfileForImageBuilderECRContainerBuilds"
+                    )
+                )
+                print("Successfully created new CodeBuild role")
+
+            task_role_name = ECS_TASK_ROLE_NAME
+            if get_context_bool(f"exists:{task_role_name}"):
+                role_arn = get_context_str(f"arn:{task_role_name}")
+                if not role_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{task_role_name}' is required if role exists."
+                    )
+                task_role = iam.Role.from_role_arn(self, "TaskRole", role_arn=role_arn)
+                print("Using existing ECS task role")
+            else:
+                task_role = iam.Role(
+                    self,
+                    "TaskRole",  # Logical ID
+                    role_name=task_role_name,  # Explicit resource name
+                    assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"),
+                )
+                for role in AWS_MANAGED_TASK_ROLES_LIST:
+                    print(f"Adding {role} to policy")
+                    task_role.add_managed_policy(
+                        iam.ManagedPolicy.from_aws_managed_policy_name(f"{role}")
+                    )
+                task_role = add_custom_policies(
+                    self, task_role, custom_policy_text=custom_sts_kms_policy
+                )
+                print("Successfully created new ECS task role")
+
+            execution_role_name = ECS_TASK_EXECUTION_ROLE_NAME
+            if get_context_bool(f"exists:{execution_role_name}"):
+                role_arn = get_context_str(f"arn:{execution_role_name}")
+                if not role_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{execution_role_name}' is required if role exists."
+                    )
+                execution_role = iam.Role.from_role_arn(
+                    self, "ExecutionRole", role_arn=role_arn
+                )
+                print("Using existing ECS execution role")
+            else:
+                execution_role = iam.Role(
+                    self,
+                    "ExecutionRole",  # Logical ID
+                    role_name=execution_role_name,  # Explicit resource name
+                    assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"),
+                )
+                for role in AWS_MANAGED_TASK_ROLES_LIST:
+                    execution_role.add_managed_policy(
+                        iam.ManagedPolicy.from_aws_managed_policy_name(f"{role}")
+                    )
+                execution_role = add_custom_policies(
+                    self, execution_role, custom_policy_text=custom_sts_kms_policy
+                )
+                print("Successfully created new ECS execution role")
+
+        except Exception as e:
+            raise Exception("Failed at IAM role step due to:", e)
+
+        # --- S3 Buckets ---
+        try:
+            log_bucket_name = S3_LOG_CONFIG_BUCKET_NAME
+            if get_context_bool(f"exists:{log_bucket_name}"):
+                bucket = s3.Bucket.from_bucket_name(
+                    self, "LogConfigBucket", bucket_name=log_bucket_name
+                )
+                print("Using existing S3 bucket", log_bucket_name)
+            else:
+                if USE_CUSTOM_KMS_KEY == "1" and isinstance(kms_key, kms.Key):
+                    bucket = s3.Bucket(
+                        self,
+                        "LogConfigBucket",
+                        bucket_name=log_bucket_name,
+                        versioned=False,
+                        removal_policy=RemovalPolicy.DESTROY,
+                        auto_delete_objects=True,
+                        encryption=s3.BucketEncryption.KMS,
+                        encryption_key=kms_key,
+                    )
+                else:
+                    bucket = s3.Bucket(
+                        self,
+                        "LogConfigBucket",
+                        bucket_name=log_bucket_name,
+                        versioned=False,
+                        removal_policy=RemovalPolicy.DESTROY,
+                        auto_delete_objects=True,
+                    )
+
+                print("Created S3 bucket", log_bucket_name)
+
+            # Add policies - this will apply to both created and imported buckets
+            # CDK handles idempotent policy additions
+            bucket.add_to_resource_policy(
+                iam.PolicyStatement(
+                    effect=iam.Effect.ALLOW,
+                    principals=[task_role],  # Pass the role object directly
+                    actions=["s3:GetObject", "s3:PutObject"],
+                    resources=[f"{bucket.bucket_arn}/*"],
+                )
+            )
+            bucket.add_to_resource_policy(
+                iam.PolicyStatement(
+                    effect=iam.Effect.ALLOW,
+                    principals=[task_role],
+                    actions=["s3:ListBucket"],
+                    resources=[bucket.bucket_arn],
+                )
+            )
+
+            output_bucket_name = S3_OUTPUT_BUCKET_NAME
+            if get_context_bool(f"exists:{output_bucket_name}"):
+                output_bucket = s3.Bucket.from_bucket_name(
+                    self, "OutputBucket", bucket_name=output_bucket_name
+                )
+                print("Using existing Output bucket", output_bucket_name)
+            else:
+                if USE_CUSTOM_KMS_KEY == "1" and isinstance(kms_key, kms.Key):
+                    output_bucket = s3.Bucket(
+                        self,
+                        "OutputBucket",
+                        bucket_name=output_bucket_name,
+                        lifecycle_rules=[
+                            s3.LifecycleRule(
+                                expiration=Duration.days(
+                                    int(DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS)
+                                )
+                            )
+                        ],
+                        versioned=False,
+                        removal_policy=RemovalPolicy.DESTROY,
+                        auto_delete_objects=True,
+                        encryption=s3.BucketEncryption.KMS,
+                        encryption_key=kms_key,
+                    )
+                else:
+                    output_bucket = s3.Bucket(
+                        self,
+                        "OutputBucket",
+                        bucket_name=output_bucket_name,
+                        lifecycle_rules=[
+                            s3.LifecycleRule(
+                                expiration=Duration.days(
+                                    int(DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS)
+                                )
+                            )
+                        ],
+                        versioned=False,
+                        removal_policy=RemovalPolicy.DESTROY,
+                        auto_delete_objects=True,
+                    )
+
+                print("Created Output bucket:", output_bucket_name)
+
+            # Add policies to output bucket
+            output_bucket.add_to_resource_policy(
+                iam.PolicyStatement(
+                    effect=iam.Effect.ALLOW,
+                    principals=[task_role],
+                    actions=["s3:GetObject", "s3:PutObject"],
+                    resources=[f"{output_bucket.bucket_arn}/*"],
+                )
+            )
+            output_bucket.add_to_resource_policy(
+                iam.PolicyStatement(
+                    effect=iam.Effect.ALLOW,
+                    principals=[task_role],
+                    actions=["s3:ListBucket"],
+                    resources=[output_bucket.bucket_arn],
+                )
+            )
+
+        except Exception as e:
+            raise Exception("Could not handle S3 buckets due to:", e)
+
+        # --- Elastic Container Registry ---
+        try:
+            full_ecr_repo_name = ECR_CDK_REPO_NAME
+            if get_context_bool(f"exists:{full_ecr_repo_name}"):
+                ecr_repo = ecr.Repository.from_repository_name(
+                    self, "ECRRepo", repository_name=full_ecr_repo_name
+                )
+                print("Using existing ECR repository")
+            else:
+                ecr_repo = ecr.Repository(
+                    self, "ECRRepo", repository_name=full_ecr_repo_name
+                )  # Explicitly set repository_name
+                print("Created ECR repository", full_ecr_repo_name)
+
+            ecr_image_loc = ecr_repo.repository_uri
+        except Exception as e:
+            raise Exception("Could not handle ECR repo due to:", e)
+
+        # --- CODEBUILD ---
+        try:
+            codebuild_project_name = CODEBUILD_PROJECT_NAME
+            if get_context_bool(f"exists:{codebuild_project_name}"):
+                # Lookup CodeBuild project by ARN from context
+                project_arn = get_context_str(f"arn:{codebuild_project_name}")
+                if not project_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{codebuild_project_name}' is required if project exists."
+                    )
+                codebuild.Project.from_project_arn(
+                    self, "CodeBuildProject", project_arn=project_arn
+                )
+                print("Using existing CodeBuild project")
+            else:
+                codebuild.Project(
+                    self,
+                    "CodeBuildProject",  # Logical ID
+                    project_name=codebuild_project_name,  # Explicit resource name
+                    role=codebuild_role,
+                    source=codebuild.Source.git_hub(
+                        owner=GITHUB_REPO_USERNAME,
+                        repo=GITHUB_REPO_NAME,
+                        branch_or_ref=GITHUB_REPO_BRANCH,
+                    ),
+                    environment=codebuild.BuildEnvironment(
+                        build_image=codebuild.LinuxBuildImage.STANDARD_7_0,
+                        privileged=True,
+                        environment_variables={
+                            "ECR_REPO_NAME": codebuild.BuildEnvironmentVariable(
+                                value=full_ecr_repo_name
+                            ),
+                            "AWS_DEFAULT_REGION": codebuild.BuildEnvironmentVariable(
+                                value=AWS_REGION
+                            ),
+                            "AWS_ACCOUNT_ID": codebuild.BuildEnvironmentVariable(
+                                value=AWS_ACCOUNT_ID
+                            ),
+                            "APP_MODE": codebuild.BuildEnvironmentVariable(
+                                value="gradio"
+                            ),
+                        },
+                    ),
+                    build_spec=codebuild.BuildSpec.from_object(
+                        {
+                            "version": "0.2",
+                            "phases": {
+                                "pre_build": {
+                                    "commands": [
+                                        "echo Logging in to Amazon ECR",
+                                        "aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com",
+                                    ]
+                                },
+                                "build": {
+                                    "commands": [
+                                        "echo Building the Docker image",
+                                        "docker build --build-args APP_MODE=$APP_MODE --target $APP_MODE -t $ECR_REPO_NAME:latest .",
+                                        "docker tag $ECR_REPO_NAME:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$ECR_REPO_NAME:latest",
+                                    ]
+                                },
+                                "post_build": {
+                                    "commands": [
+                                        "echo Pushing the Docker image",
+                                        "docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$ECR_REPO_NAME:latest",
+                                    ]
+                                },
+                            },
+                        }
+                    ),
+                )
+                print("Successfully created CodeBuild project", codebuild_project_name)
+
+            # Imported projects have role=undefined in CDK; use the actual service
+            # role from context (existing project) or the managed codebuild_role (new).
+            if get_context_bool(f"exists:{codebuild_project_name}"):
+                project_service_role_arn = get_context_str(
+                    f"service_role_arn:{codebuild_project_name}"
+                )
+                if project_service_role_arn:
+                    ecr_grantee = iam.Role.from_role_arn(
+                        self,
+                        "CodeBuildProjectServiceRole",
+                        role_arn=project_service_role_arn,
+                        mutable=True,
+                    )
+                else:
+                    ecr_grantee = codebuild_role
+            else:
+                ecr_grantee = codebuild_role
+            ecr_repo.grant_pull_push(ecr_grantee)
+
+        except Exception as e:
+            raise Exception("Could not handle Codebuild project due to:", e)
+
+        # --- Security Groups ---
+        try:
+            ecs_security_group_name = ECS_SECURITY_GROUP_NAME
+
+            try:
+                ecs_security_group = ec2.SecurityGroup(
+                    self,
+                    "ECSSecurityGroup",  # Logical ID
+                    security_group_name=ecs_security_group_name,  # Explicit resource name
+                    vpc=vpc,
+                )
+                print(f"Created Security Group: {ecs_security_group_name}")
+            except Exception as e:  # If lookup fails, create
+                print("Failed to create ECS security group due to:", e)
+
+            alb_security_group_name = ALB_NAME_SECURITY_GROUP_NAME
+
+            try:
+                alb_security_group = ec2.SecurityGroup(
+                    self,
+                    "ALBSecurityGroup",  # Logical ID
+                    security_group_name=alb_security_group_name,  # Explicit resource name
+                    vpc=vpc,
+                )
+                print(f"Created Security Group: {alb_security_group_name}")
+            except Exception as e:  # If lookup fails, create
+                print("Failed to create ALB security group due to:", e)
+
+            # Define Ingress Rules - CDK will manage adding/removing these as needed
+            ec2_port_gradio_server_port = ec2.Port.tcp(
+                int(GRADIO_SERVER_PORT)
+            )  # Ensure port is int
+            ecs_security_group.add_ingress_rule(
+                peer=alb_security_group,
+                connection=ec2_port_gradio_server_port,
+                description="ALB traffic",
+            )
+
+            alb_security_group.add_ingress_rule(
+                peer=ec2.Peer.prefix_list("pl-93a247fa"),
+                connection=ec2.Port.all_traffic(),
+                description="CloudFront traffic",
+            )
+
+        except Exception as e:
+            raise Exception("Could not handle security groups due to:", e)
+
+        # --- DynamoDB tables for logs (optional) ---
+
+        if SAVE_LOGS_TO_DYNAMODB == "True":
+            try:
+                print("Creating DynamoDB tables for logs")
+
+                dynamodb.Table(
+                    self,
+                    "RedactionAccessDataTable",
+                    table_name=ACCESS_LOG_DYNAMODB_TABLE_NAME,
+                    partition_key=dynamodb.Attribute(
+                        name="id", type=dynamodb.AttributeType.STRING
+                    ),
+                    billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
+                    removal_policy=RemovalPolicy.DESTROY,
+                )
+
+                dynamodb.Table(
+                    self,
+                    "RedactionFeedbackDataTable",
+                    table_name=FEEDBACK_LOG_DYNAMODB_TABLE_NAME,
+                    partition_key=dynamodb.Attribute(
+                        name="id", type=dynamodb.AttributeType.STRING
+                    ),
+                    billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
+                    removal_policy=RemovalPolicy.DESTROY,
+                )
+
+                dynamodb.Table(
+                    self,
+                    "RedactionUsageDataTable",
+                    table_name=USAGE_LOG_DYNAMODB_TABLE_NAME,
+                    partition_key=dynamodb.Attribute(
+                        name="id", type=dynamodb.AttributeType.STRING
+                    ),
+                    billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
+                    removal_policy=RemovalPolicy.DESTROY,
+                )
+
+            except Exception as e:
+                raise Exception("Could not create DynamoDB tables due to:", e)
+
+        # --- ALB ---
+        try:
+            load_balancer_name = ALB_NAME
+            if len(load_balancer_name) > 32:
+                load_balancer_name = load_balancer_name[-32:]
+            alb_arn = get_context_str(f"arn:{load_balancer_name}") or (
+                EXISTING_LOAD_BALANCER_ARN or None
+            )
+            alb_dns_name = get_context_str(f"dns:{load_balancer_name}") or (
+                EXISTING_LOAD_BALANCER_DNS or None
+            )
+            if alb_arn and alb_dns_name:
+                alb_security_group_id = (
+                    get_context_str(f"security_group_id:{load_balancer_name}")
+                    or alb_security_group.security_group_id
+                )
+                alb_attrs = {
+                    "load_balancer_arn": alb_arn,
+                    "load_balancer_dns_name": alb_dns_name,
+                    "security_group_id": alb_security_group_id,
+                    "vpc": vpc,
+                }
+                alb_canonical_zone_id = get_context_str(
+                    f"canonical_hosted_zone_id:{load_balancer_name}"
+                )
+                if alb_canonical_zone_id:
+                    alb_attrs["load_balancer_canonical_hosted_zone_id"] = (
+                        alb_canonical_zone_id
+                    )
+                alb = elbv2.ApplicationLoadBalancer.from_application_load_balancer_attributes(
+                    self,
+                    "ALB",
+                    **alb_attrs,
+                )
+                print(f"Using existing Application Load Balancer {load_balancer_name}.")
+            else:
+                alb = elbv2.ApplicationLoadBalancer(
+                    self,
+                    "ALB",  # Logical ID
+                    load_balancer_name=load_balancer_name,  # Explicit resource name
+                    vpc=vpc,
+                    internet_facing=True,
+                    security_group=alb_security_group,  # Link to SG
+                    vpc_subnets=public_subnet_selection,  # Link to subnets
+                )
+                print("Successfully created new Application Load Balancer")
+        except Exception as e:
+            raise Exception("Could not handle application load balancer due to:", e)
+
+        # --- Cognito User Pool ---
+        try:
+            if get_context_bool(f"exists:{COGNITO_USER_POOL_NAME}"):
+                # Lookup by ID from context
+                user_pool_id = get_context_str(f"id:{COGNITO_USER_POOL_NAME}")
+                if not user_pool_id:
+                    raise ValueError(
+                        f"Context value 'id:{COGNITO_USER_POOL_NAME}' is required if User Pool exists."
+                    )
+                user_pool = cognito.UserPool.from_user_pool_id(
+                    self, "UserPool", user_pool_id=user_pool_id
+                )
+                print(f"Using existing user pool {user_pool_id}.")
+            else:
+                user_pool = cognito.UserPool(
+                    self,
+                    "UserPool",
+                    user_pool_name=COGNITO_USER_POOL_NAME,
+                    mfa=cognito.Mfa.OFF,  # Adjust as needed
+                    sign_in_aliases=cognito.SignInAliases(email=True),
+                    removal_policy=RemovalPolicy.DESTROY,
+                )  # Adjust as needed
+                print(f"Created new user pool {user_pool.user_pool_id}.")
+
+            # If you're using a certificate, assume that you will be using the ALB Cognito login features. You need different redirect URLs to accept the token that comes from Cognito authentication.
+            if ACM_SSL_CERTIFICATE_ARN:
+                redirect_uris = [
+                    COGNITO_REDIRECTION_URL,
+                    COGNITO_REDIRECTION_URL + "/oauth2/idpresponse",
+                ]
+            else:
+                redirect_uris = [COGNITO_REDIRECTION_URL]
+
+            user_pool_client_name = COGNITO_USER_POOL_CLIENT_NAME
+            if get_context_bool(f"exists:{user_pool_client_name}"):
+                # Lookup by ID from context (requires User Pool object)
+                user_pool_client_id = get_context_str(f"id:{user_pool_client_name}")
+                if not user_pool_client_id:
+                    raise ValueError(
+                        f"Context value 'id:{user_pool_client_name}' is required if User Pool Client exists."
+                    )
+                user_pool_client = cognito.UserPoolClient.from_user_pool_client_id(
+                    self, "UserPoolClient", user_pool_client_id=user_pool_client_id
+                )
+                print(f"Using existing user pool client {user_pool_client_id}.")
+            else:
+                user_pool_client = cognito.UserPoolClient(
+                    self,
+                    "UserPoolClient",
+                    auth_flows=cognito.AuthFlow(
+                        user_srp=True, user_password=True
+                    ),  # Example: enable SRP for secure sign-in
+                    user_pool=user_pool,
+                    generate_secret=True,
+                    user_pool_client_name=user_pool_client_name,
+                    supported_identity_providers=[
+                        cognito.UserPoolClientIdentityProvider.COGNITO
+                    ],
+                    o_auth=cognito.OAuthSettings(
+                        flows=cognito.OAuthFlows(authorization_code_grant=True),
+                        scopes=[
+                            cognito.OAuthScope.OPENID,
+                            cognito.OAuthScope.EMAIL,
+                            cognito.OAuthScope.PROFILE,
+                        ],
+                        callback_urls=redirect_uris,
+                    ),
+                    refresh_token_validity=Duration.minutes(
+                        COGNITO_REFRESH_TOKEN_VALIDITY
+                    ),
+                    id_token_validity=Duration.minutes(COGNITO_ID_TOKEN_VALIDITY),
+                    access_token_validity=Duration.minutes(
+                        COGNITO_ACCESS_TOKEN_VALIDITY
+                    ),
+                )
+
+            CfnOutput(
+                self, "CognitoAppClientId", value=user_pool_client.user_pool_client_id
+            )
+
+            print(
+                f"Created new user pool client {user_pool_client.user_pool_client_id}."
+            )
+
+            # Add a domain to the User Pool (crucial for ALB integration)
+            user_pool_domain = user_pool.add_domain(
+                "UserPoolDomain",
+                cognito_domain=cognito.CognitoDomainOptions(
+                    domain_prefix=COGNITO_USER_POOL_DOMAIN_PREFIX
+                ),
+            )
+
+            # Apply removal_policy to the created UserPoolDomain construct
+            user_pool_domain.apply_removal_policy(policy=RemovalPolicy.DESTROY)
+
+            CfnOutput(
+                self, "CognitoUserPoolLoginUrl", value=user_pool_domain.base_url()
+            )
+
+        except Exception as e:
+            raise Exception("Could not handle Cognito resources due to:", e)
+
+        # --- Secrets Manager Secret ---
+        try:
+            secret_name = COGNITO_USER_POOL_CLIENT_SECRET_NAME
+            if get_context_bool(f"exists:{secret_name}"):
+                # Lookup by name
+                secret = secretsmanager.Secret.from_secret_name_v2(
+                    self, "CognitoSecret", secret_name=secret_name
+                )
+                print("Using existing Secret.")
+            else:
+                if USE_CUSTOM_KMS_KEY == "1" and isinstance(kms_key, kms.Key):
+                    secret = secretsmanager.Secret(
+                        self,
+                        "CognitoSecret",  # Logical ID
+                        secret_name=secret_name,  # Explicit resource name
+                        secret_object_value={
+                            "REDACTION_USER_POOL_ID": SecretValue.unsafe_plain_text(
+                                user_pool.user_pool_id
+                            ),  # Use the CDK attribute
+                            "REDACTION_CLIENT_ID": SecretValue.unsafe_plain_text(
+                                user_pool_client.user_pool_client_id
+                            ),  # Use the CDK attribute
+                            "REDACTION_CLIENT_SECRET": user_pool_client.user_pool_client_secret,  # Use the CDK attribute
+                        },
+                        encryption_key=kms_key,
+                    )
+                else:
+                    secret = secretsmanager.Secret(
+                        self,
+                        "CognitoSecret",  # Logical ID
+                        secret_name=secret_name,  # Explicit resource name
+                        secret_object_value={
+                            "REDACTION_USER_POOL_ID": SecretValue.unsafe_plain_text(
+                                user_pool.user_pool_id
+                            ),  # Use the CDK attribute
+                            "REDACTION_CLIENT_ID": SecretValue.unsafe_plain_text(
+                                user_pool_client.user_pool_client_id
+                            ),  # Use the CDK attribute
+                            "REDACTION_CLIENT_SECRET": user_pool_client.user_pool_client_secret,  # Use the CDK attribute
+                        },
+                    )
+
+                print(
+                    "Created new secret in Secrets Manager for Cognito user pool and related details."
+                )
+
+        except Exception as e:
+            raise Exception("Could not handle Secrets Manager secret due to:", e)
+
+        # --- Fargate Task Definition ---
+        try:
+            fargate_task_definition_name = FARGATE_TASK_DEFINITION_NAME
+
+            read_only_file_system = ECS_READ_ONLY_FILE_SYSTEM == "True"
+
+            if os.path.exists(TASK_DEFINITION_FILE_LOCATION):
+                with open(TASK_DEFINITION_FILE_LOCATION) as f:  # Use correct path
+                    task_def_params = json.load(f)
+                # Need to ensure taskRoleArn and executionRoleArn in JSON are correct ARN strings
+            else:
+                epheremal_storage_volume_name = "appEphemeralVolume"
+
+                task_def_params = {}
+                task_def_params["taskRoleArn"] = (
+                    task_role.role_arn
+                )  # Use CDK role object ARN
+                task_def_params["executionRoleArn"] = (
+                    execution_role.role_arn
+                )  # Use CDK role object ARN
+                task_def_params["memory"] = ECS_TASK_MEMORY_SIZE
+                task_def_params["cpu"] = ECS_TASK_CPU_SIZE
+                container_def = {
+                    "name": full_ecr_repo_name,
+                    "image": ecr_image_loc + ":latest",
+                    "essential": True,
+                    "portMappings": [
+                        {
+                            "containerPort": int(GRADIO_SERVER_PORT),
+                            "hostPort": int(GRADIO_SERVER_PORT),
+                            "protocol": "tcp",
+                            "appProtocol": "http",
+                        }
+                    ],
+                    "logConfiguration": {
+                        "logDriver": "awslogs",
+                        "options": {
+                            "awslogs-group": ECS_LOG_GROUP_NAME,
+                            "awslogs-region": AWS_REGION,
+                            "awslogs-stream-prefix": "ecs",
+                        },
+                    },
+                    "environmentFiles": [
+                        {"value": bucket.bucket_arn + "/config.env", "type": "s3"}
+                    ],
+                    "memoryReservation": int(task_def_params["memory"])
+                    - 512,  # Reserve some memory for the container
+                    "mountPoints": [
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/logs",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/feedback",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/usage",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/input",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/output",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/tmp",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/config",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/tmp/matplotlib_cache",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/tmp",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/var/tmp",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/tmp/tld",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/tmp/gradio_tmp",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/.paddlex",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/.local/share/spacy/data",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/usr/share/tessdata",
+                            "readOnly": False,
+                        },
+                    ],
+                    "readonlyRootFilesystem": read_only_file_system,
+                }
+                task_def_params["containerDefinitions"] = [container_def]
+
+            log_group_name_from_config = task_def_params["containerDefinitions"][0][
+                "logConfiguration"
+            ]["options"]["awslogs-group"]
+
+            cdk_managed_log_group = logs.LogGroup(
+                self,
+                "MyTaskLogGroup",  # CDK Logical ID
+                log_group_name=log_group_name_from_config,
+                retention=logs.RetentionDays.ONE_MONTH,
+                removal_policy=RemovalPolicy.DESTROY,
+            )
+
+            epheremal_storage_volume_cdk_obj = ecs.Volume(
+                name=epheremal_storage_volume_name
+            )
+
+            fargate_task_definition = ecs.FargateTaskDefinition(
+                self,
+                "FargateTaskDefinition",  # Logical ID
+                family=fargate_task_definition_name,
+                cpu=int(task_def_params["cpu"]),
+                memory_limit_mib=int(task_def_params["memory"]),
+                task_role=task_role,
+                execution_role=execution_role,
+                runtime_platform=ecs.RuntimePlatform(
+                    cpu_architecture=ecs.CpuArchitecture.X86_64,
+                    operating_system_family=ecs.OperatingSystemFamily.LINUX,
+                ),
+                ephemeral_storage_gib=21,  # Minimum is 21 GiB
+                volumes=[epheremal_storage_volume_cdk_obj],
+            )
+            print("Fargate task definition defined.")
+
+            # Add container definitions to the task definition object
+            if task_def_params["containerDefinitions"]:
+                container_def_params = task_def_params["containerDefinitions"][0]
+
+                if container_def_params.get("environmentFiles"):
+                    env_files = []
+                    for env_file_param in container_def_params["environmentFiles"]:
+                        # Need to parse the ARN to get the bucket object and key
+                        env_file_arn_parts = env_file_param["value"].split(":::")
+                        bucket_name_and_key = env_file_arn_parts[-1]
+                        env_bucket_name, env_key = bucket_name_and_key.split("/", 1)
+
+                        env_file = ecs.EnvironmentFile.from_bucket(bucket, env_key)
+
+                        env_files.append(env_file)
+
+                container = fargate_task_definition.add_container(
+                    container_def_params["name"],
+                    image=ecs.ContainerImage.from_registry(
+                        container_def_params["image"]
+                    ),
+                    logging=ecs.LogDriver.aws_logs(
+                        stream_prefix=container_def_params["logConfiguration"][
+                            "options"
+                        ]["awslogs-stream-prefix"],
+                        log_group=cdk_managed_log_group,
+                    ),
+                    secrets={
+                        "AWS_USER_POOL_ID": ecs.Secret.from_secrets_manager(
+                            secret, "REDACTION_USER_POOL_ID"
+                        ),
+                        "AWS_CLIENT_ID": ecs.Secret.from_secrets_manager(
+                            secret, "REDACTION_CLIENT_ID"
+                        ),
+                        "AWS_CLIENT_SECRET": ecs.Secret.from_secrets_manager(
+                            secret, "REDACTION_CLIENT_SECRET"
+                        ),
+                    },
+                    environment_files=env_files,
+                    readonly_root_filesystem=read_only_file_system,
+                )
+
+                for port_mapping in container_def_params["portMappings"]:
+                    container.add_port_mappings(
+                        ecs.PortMapping(
+                            container_port=int(port_mapping["containerPort"]),
+                            host_port=int(port_mapping["hostPort"]),
+                            name="port-" + str(port_mapping["containerPort"]),
+                            app_protocol=ecs.AppProtocol.http,
+                            protocol=ecs.Protocol.TCP,
+                        )
+                    )
+
+                container.add_port_mappings(
+                    ecs.PortMapping(
+                        container_port=80,
+                        host_port=80,
+                        name="port-80",
+                        app_protocol=ecs.AppProtocol.http,
+                        protocol=ecs.Protocol.TCP,
+                    )
+                )
+
+                if container_def_params.get("mountPoints"):
+                    mount_points = []
+                    for mount_point in container_def_params["mountPoints"]:
+                        mount_points.append(
+                            ecs.MountPoint(
+                                container_path=mount_point["containerPath"],
+                                read_only=mount_point["readOnly"],
+                                source_volume=epheremal_storage_volume_name,
+                            )
+                        )
+                    container.add_mount_points(*mount_points)
+
+        except Exception as e:
+            raise Exception("Could not handle Fargate task definition due to:", e)
+
+        # --- ECS Cluster ---
+        try:
+            cluster = ecs.Cluster(
+                self,
+                "ECSCluster",  # Logical ID
+                cluster_name=CLUSTER_NAME,  # Explicit resource name
+                enable_fargate_capacity_providers=True,
+                vpc=vpc,
+            )
+            print("Successfully created new ECS cluster")
+        except Exception as e:
+            raise Exception("Could not handle ECS cluster due to:", e)
+
+        # --- ECS Service ---
+        try:
+            ecs_service_name = ECS_SERVICE_NAME
+
+            if ECS_USE_FARGATE_SPOT == "True":
+                use_fargate_spot = "FARGATE_SPOT"
+            if ECS_USE_FARGATE_SPOT == "False":
+                use_fargate_spot = "FARGATE"
+
+            # Check if service exists - from_service_arn or from_service_name (needs cluster)
+            try:
+                # from_service_name is useful if you have the cluster object
+                ecs_service = ecs.FargateService.from_service_attributes(
+                    self,
+                    "ECSService",  # Logical ID
+                    cluster=cluster,  # Requires the cluster object
+                    service_name=ecs_service_name,
+                )
+                print(f"Using existing ECS service {ecs_service_name}.")
+            except Exception:
+                # Service will be created with a count of 0, because you haven't yet actually built the initial Docker container with CodeBuild
+                ecs_service = ecs.FargateService(
+                    self,
+                    "ECSService",  # Logical ID
+                    service_name=ecs_service_name,  # Explicit resource name
+                    platform_version=ecs.FargatePlatformVersion.LATEST,
+                    capacity_provider_strategies=[
+                        ecs.CapacityProviderStrategy(
+                            capacity_provider=use_fargate_spot, base=0, weight=1
+                        )
+                    ],
+                    cluster=cluster,
+                    task_definition=fargate_task_definition,  # Link to TD
+                    security_groups=[ecs_security_group],  # Link to SG
+                    vpc_subnets=ec2.SubnetSelection(
+                        subnets=self.private_subnets
+                    ),  # Link to subnets
+                    min_healthy_percent=0,
+                    max_healthy_percent=100,
+                    desired_count=0,
+                )
+                print("Successfully created new ECS service")
+
+            # Note: Auto-scaling setup would typically go here if needed for the service
+
+        except Exception as e:
+            raise Exception("Could not handle ECS service due to:", e)
+
+        # --- Grant Secret Read Access (Applies to both created and imported roles) ---
+        try:
+            secret.grant_read(task_role)
+            secret.grant_read(execution_role)
+        except Exception as e:
+            raise Exception("Could not grant access to Secrets Manager due to:", e)
+
+        # --- ALB TARGET GROUPS AND LISTENERS ---
+        # This section should primarily define the resources if they are managed by this stack.
+        # CDK handles adding/removing targets and actions on updates.
+        # If they might pre-exist outside the stack, you need lookups.
+        cookie_duration = Duration.hours(12)
+        target_group_name = ALB_TARGET_GROUP_NAME  # Explicit resource name
+        cloudfront_distribution_url = "cloudfront_placeholder.net"  # Need to replace this afterwards with the actual cloudfront_distribution.domain_name
+
+        try:
+            # --- CREATING TARGET GROUPS AND ADDING THE CLOUDFRONT LISTENER RULE ---
+
+            target_group = elbv2.ApplicationTargetGroup(
+                self,
+                "AppTargetGroup",  # Logical ID
+                target_group_name=target_group_name,  # Explicit resource name
+                port=int(GRADIO_SERVER_PORT),  # Ensure port is int
+                protocol=elbv2.ApplicationProtocol.HTTP,
+                targets=[ecs_service],  # Link to ECS Service
+                stickiness_cookie_duration=cookie_duration,
+                vpc=vpc,  # Target Groups need VPC
+            )
+            print(f"ALB target group {target_group_name} defined.")
+
+            # First HTTP
+            listener_port = 80
+            # Check if Listener exists - from_listener_arn or lookup by port/ALB
+
+            http_listener = alb.add_listener(
+                "HttpListener",  # Logical ID
+                port=listener_port,
+                open=False,  # Be cautious with open=True, usually restrict source SG
+            )
+            print(f"ALB listener on port {listener_port} defined.")
+
+            if ACM_SSL_CERTIFICATE_ARN:
+                http_listener.add_action(
+                    "DefaultAction",  # Logical ID for the default action
+                    action=elbv2.ListenerAction.redirect(
+                        protocol="HTTPS",
+                        host="#{host}",
+                        port="443",
+                        path="/#{path}",
+                        query="#{query}",
+                    ),
+                )
+            else:
+                if USE_CLOUDFRONT == "True":
+
+                    # The following default action can be added for the listener after a host header rule is added to the listener manually in the Console as suggested in the above comments.
+                    http_listener.add_action(
+                        "DefaultAction",  # Logical ID for the default action
+                        action=elbv2.ListenerAction.fixed_response(
+                            status_code=403,
+                            content_type="text/plain",
+                            message_body="Access denied",
+                        ),
+                    )
+
+                    # Add the Listener Rule for the specific CloudFront Host Header
+                    http_listener.add_action(
+                        "CloudFrontHostHeaderRule",
+                        action=elbv2.ListenerAction.forward(
+                            target_groups=[target_group],
+                            stickiness_duration=cookie_duration,
+                        ),
+                        priority=1,  # Example priority. Adjust as needed. Lower is evaluated first.
+                        conditions=[
+                            elbv2.ListenerCondition.host_headers(
+                                [cloudfront_distribution_url]
+                            )  # May have to redefine url in console afterwards if not specified in config file
+                        ],
+                    )
+
+                else:
+                    # Add the Listener Rule for the specific CloudFront Host Header
+                    http_listener.add_action(
+                        "CloudFrontHostHeaderRule",
+                        action=elbv2.ListenerAction.forward(
+                            target_groups=[target_group],
+                            stickiness_duration=cookie_duration,
+                        ),
+                    )
+
+                print("Added targets and actions to ALB HTTP listener.")
+
+            # Now the same for HTTPS if you have an ACM certificate
+            if ACM_SSL_CERTIFICATE_ARN:
+                listener_port_https = 443
+                # Check if Listener exists - from_listener_arn or lookup by port/ALB
+
+                https_listener = add_alb_https_listener_with_cert(
+                    self,
+                    "MyHttpsListener",  # Logical ID for the HTTPS listener
+                    alb,
+                    acm_certificate_arn=ACM_SSL_CERTIFICATE_ARN,
+                    default_target_group=target_group,
+                    enable_cognito_auth=True,
+                    cognito_user_pool=user_pool,
+                    cognito_user_pool_client=user_pool_client,
+                    cognito_user_pool_domain=user_pool_domain,
+                    listener_open_to_internet=True,
+                    stickiness_cookie_duration=cookie_duration,
+                )
+
+                if https_listener:
+                    CfnOutput(
+                        self, "HttpsListenerArn", value=https_listener.listener_arn
+                    )
+
+                print(f"ALB listener on port {listener_port_https} defined.")
+
+                # if USE_CLOUDFRONT == 'True':
+                #     # Add default action to the listener
+                #     https_listener.add_action(
+                #         "DefaultAction", # Logical ID for the default action
+                #         action=elbv2.ListenerAction.fixed_response(
+                #             status_code=403,
+                #             content_type="text/plain",
+                #             message_body="Access denied",
+                #         ),
+                #     )
+
+                #     # Add the Listener Rule for the specific CloudFront Host Header
+                #     https_listener.add_action(
+                #         "CloudFrontHostHeaderRuleHTTPS",
+                #         action=elbv2.ListenerAction.forward(target_groups=[target_group],stickiness_duration=cookie_duration),
+                #         priority=1, # Example priority. Adjust as needed. Lower is evaluated first.
+                #         conditions=[
+                #             elbv2.ListenerCondition.host_headers([cloudfront_distribution_url])
+                #         ]
+                #     )
+                # else:
+                #     https_listener.add_action(
+                #         "CloudFrontHostHeaderRuleHTTPS",
+                #         action=elbv2.ListenerAction.forward(target_groups=[target_group],stickiness_duration=cookie_duration))
+
+                print("Added targets and actions to ALB HTTPS listener.")
+
+        except Exception as e:
+            raise Exception(
+                "Could not handle ALB target groups and listeners due to:", e
+            )
+
+        # Create WAF to attach to load balancer
+        try:
+            web_acl_name = LOAD_BALANCER_WEB_ACL_NAME
+            if get_context_bool(f"exists:{web_acl_name}"):
+                # Lookup WAF ACL by ARN from context
+                web_acl_arn = get_context_str(f"arn:{web_acl_name}")
+                if not web_acl_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{web_acl_name}' is required if Web ACL exists."
+                    )
+
+                web_acl = create_web_acl_with_common_rules(
+                    self, web_acl_name, waf_scope="REGIONAL"
+                )  # Assuming it takes scope and name
+                print(f"Handled ALB WAF web ACL {web_acl_name}.")
+            else:
+                web_acl = create_web_acl_with_common_rules(
+                    self, web_acl_name, waf_scope="REGIONAL"
+                )  # Assuming it takes scope and name
+                print(f"Created ALB WAF web ACL {web_acl_name}.")
+
+            wafv2.CfnWebACLAssociation(
+                self,
+                id="alb_waf_association",
+                resource_arn=alb.load_balancer_arn,
+                web_acl_arn=web_acl.attr_arn,
+            )
+
+        except Exception as e:
+            raise Exception("Could not handle create ALB WAF web ACL due to:", e)
+
+        # --- Outputs for other stacks/regions ---
+
+        self.params = dict()
+        self.params["alb_arn_output"] = alb.load_balancer_arn
+        self.params["alb_security_group_id"] = alb_security_group.security_group_id
+        self.params["alb_dns_name"] = alb.load_balancer_dns_name
+
+        CfnOutput(
+            self,
+            "AlbArnOutput",
+            value=alb.load_balancer_arn,
+            description="ARN of the Application Load Balancer",
+            export_name=f"{self.stack_name}-AlbArn",
+        )  # Export name must be unique within the account/region
+
+        CfnOutput(
+            self,
+            "AlbSecurityGroupIdOutput",
+            value=alb_security_group.security_group_id,
+            description="ID of the ALB's Security Group",
+            export_name=f"{self.stack_name}-AlbSgId",
+        )
+        CfnOutput(self, "ALBName", value=load_balancer_name)
+
+        CfnOutput(self, "RegionalAlbDnsName", value=alb.load_balancer_dns_name)
+
+        CfnOutput(self, "CognitoPoolId", value=user_pool.user_pool_id)
+        # Add other outputs if needed
+
+        CfnOutput(self, "ECRRepoUri", value=ecr_repo.repository_uri)
+
+
+# --- CLOUDFRONT DISTRIBUTION in separate stack (us-east-1 required) ---
+class CdkStackCloudfront(Stack):
+
+    def __init__(
+        self,
+        scope: Construct,
+        construct_id: str,
+        alb_arn: str,
+        alb_sec_group_id: str,
+        alb_dns_name: str,
+        **kwargs,
+    ) -> None:
+        super().__init__(scope, construct_id, **kwargs)
+
+        # --- Helper to get context values ---
+        def get_context_bool(key: str, default: bool = False) -> bool:
+            return self.node.try_get_context(key) or default
+
+        def get_context_str(key: str, default: str = None) -> str:
+            return self.node.try_get_context(key) or default
+
+        def get_context_dict(scope: Construct, key: str, default: dict = None) -> dict:
+            return scope.node.try_get_context(key) or default
+
+        print(f"CloudFront Stack: Received ALB ARN: {alb_arn}")
+        print(f"CloudFront Stack: Received ALB Security Group ID: {alb_sec_group_id}")
+
+        if not alb_arn:
+            raise ValueError("ALB ARN must be provided to CloudFront stack")
+        if not alb_sec_group_id:
+            raise ValueError(
+                "ALB Security Group ID must be provided to CloudFront stack"
+            )
+
+        # 2. Import the ALB using its ARN
+        # This imports an existing ALB as a construct in the CloudFront stack's context.
+        # CloudFormation will understand this reference at deploy time.
+        alb = elbv2.ApplicationLoadBalancer.from_application_load_balancer_attributes(
+            self,
+            "ImportedAlb",
+            load_balancer_arn=alb_arn,
+            security_group_id=alb_sec_group_id,
+            load_balancer_dns_name=alb_dns_name,
+        )
+
+        try:
+            web_acl_name = WEB_ACL_NAME
+            if get_context_bool(f"exists:{web_acl_name}"):
+                # Lookup WAF ACL by ARN from context
+                web_acl_arn = get_context_str(f"arn:{web_acl_name}")
+                if not web_acl_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{web_acl_name}' is required if Web ACL exists."
+                    )
+
+                web_acl = create_web_acl_with_common_rules(
+                    self, web_acl_name
+                )  # Assuming it takes scope and name
+                print(f"Handled Cloudfront WAF web ACL {web_acl_name}.")
+            else:
+                web_acl = create_web_acl_with_common_rules(
+                    self, web_acl_name
+                )  # Assuming it takes scope and name
+                print(f"Created Cloudfront WAF web ACL {web_acl_name}.")
+
+            # Add ALB as CloudFront Origin
+            origin = origins.LoadBalancerV2Origin(
+                alb,  # Use the created or looked-up ALB object
+                custom_headers={CUSTOM_HEADER: CUSTOM_HEADER_VALUE},
+                origin_shield_enabled=False,
+                protocol_policy=cloudfront.OriginProtocolPolicy.HTTP_ONLY,
+            )
+
+            if CLOUDFRONT_GEO_RESTRICTION:
+                geo_restrict = cloudfront.GeoRestriction.allowlist(
+                    CLOUDFRONT_GEO_RESTRICTION
+                )
+            else:
+                geo_restrict = None
+
+            cloudfront_distribution = cloudfront.Distribution(
+                self,
+                "CloudFrontDistribution",  # Logical ID
+                comment=CLOUDFRONT_DISTRIBUTION_NAME,  # Use name as comment for easier identification
+                geo_restriction=geo_restrict,
+                default_behavior=cloudfront.BehaviorOptions(
+                    origin=origin,
+                    viewer_protocol_policy=cloudfront.ViewerProtocolPolicy.REDIRECT_TO_HTTPS,
+                    allowed_methods=cloudfront.AllowedMethods.ALLOW_ALL,
+                    cache_policy=cloudfront.CachePolicy.CACHING_DISABLED,
+                    origin_request_policy=cloudfront.OriginRequestPolicy.ALL_VIEWER,
+                ),
+                web_acl_id=web_acl.attr_arn,
+            )
+            print(f"Cloudfront distribution {CLOUDFRONT_DISTRIBUTION_NAME} defined.")
+
+        except Exception as e:
+            raise Exception("Could not handle Cloudfront distribution due to:", e)
+
+        # --- Outputs ---
+        CfnOutput(
+            self, "CloudFrontDistributionURL", value=cloudfront_distribution.domain_name
+        )

cdk/check_resources.py

@@ -0,0 +1,400 @@
+import json
+import os
+from typing import Dict, List
+
+from cdk_config import (  # Import necessary config
+    ALB_NAME,
+    AWS_REGION,
+    CDK_CONFIG_PATH,
+    CDK_FOLDER,
+    CODEBUILD_PROJECT_NAME,
+    CODEBUILD_ROLE_NAME,
+    COGNITO_USER_POOL_CLIENT_NAME,
+    COGNITO_USER_POOL_CLIENT_SECRET_NAME,
+    COGNITO_USER_POOL_NAME,
+    CONTEXT_FILE,
+    ECR_CDK_REPO_NAME,
+    ECS_TASK_EXECUTION_ROLE_NAME,
+    ECS_TASK_ROLE_NAME,
+    PRIVATE_SUBNET_AVAILABILITY_ZONES,
+    PRIVATE_SUBNET_CIDR_BLOCKS,
+    PRIVATE_SUBNETS_TO_USE,
+    PUBLIC_SUBNET_AVAILABILITY_ZONES,
+    PUBLIC_SUBNET_CIDR_BLOCKS,
+    PUBLIC_SUBNETS_TO_USE,
+    S3_LOG_CONFIG_BUCKET_NAME,
+    S3_OUTPUT_BUCKET_NAME,
+    VPC_NAME,
+    WEB_ACL_NAME,
+)
+from cdk_functions import (  # Import your check functions (assuming they use Boto3)
+    _get_existing_subnets_in_vpc,
+    check_alb_exists,
+    check_codebuild_project_exists,
+    check_ecr_repo_exists,
+    check_for_existing_role,
+    check_for_existing_user_pool,
+    check_for_existing_user_pool_client,
+    check_for_secret,
+    check_s3_bucket_exists,
+    check_subnet_exists_by_name,
+    check_web_acl_exists,
+    get_vpc_id_by_name,
+    validate_subnet_creation_parameters,
+    # Add other check functions as needed
+)
+
+cdk_folder = CDK_FOLDER  # <FULL_PATH_TO_CDK_FOLDER_HERE>
+
+# Full path needed to find config file
+os.environ["CDK_CONFIG_PATH"] = cdk_folder + CDK_CONFIG_PATH
+
+
+# --- Helper to parse environment variables into lists ---
+def _get_env_list(env_var_name: str) -> List[str]:
+    """Parses a comma-separated environment variable into a list of strings."""
+    value = env_var_name[1:-1].strip().replace('"', "").replace("'", "")
+    if not value:
+        return []
+    # Split by comma and filter out any empty strings that might result from extra commas
+    return [s.strip() for s in value.split(",") if s.strip()]
+
+
+if PUBLIC_SUBNETS_TO_USE and not isinstance(PUBLIC_SUBNETS_TO_USE, list):
+    PUBLIC_SUBNETS_TO_USE = _get_env_list(PUBLIC_SUBNETS_TO_USE)
+if PRIVATE_SUBNETS_TO_USE and not isinstance(PRIVATE_SUBNETS_TO_USE, list):
+    PRIVATE_SUBNETS_TO_USE = _get_env_list(PRIVATE_SUBNETS_TO_USE)
+if PUBLIC_SUBNET_CIDR_BLOCKS and not isinstance(PUBLIC_SUBNET_CIDR_BLOCKS, list):
+    PUBLIC_SUBNET_CIDR_BLOCKS = _get_env_list(PUBLIC_SUBNET_CIDR_BLOCKS)
+if PUBLIC_SUBNET_AVAILABILITY_ZONES and not isinstance(
+    PUBLIC_SUBNET_AVAILABILITY_ZONES, list
+):
+    PUBLIC_SUBNET_AVAILABILITY_ZONES = _get_env_list(PUBLIC_SUBNET_AVAILABILITY_ZONES)
+if PRIVATE_SUBNET_CIDR_BLOCKS and not isinstance(PRIVATE_SUBNET_CIDR_BLOCKS, list):
+    PRIVATE_SUBNET_CIDR_BLOCKS = _get_env_list(PRIVATE_SUBNET_CIDR_BLOCKS)
+if PRIVATE_SUBNET_AVAILABILITY_ZONES and not isinstance(
+    PRIVATE_SUBNET_AVAILABILITY_ZONES, list
+):
+    PRIVATE_SUBNET_AVAILABILITY_ZONES = _get_env_list(PRIVATE_SUBNET_AVAILABILITY_ZONES)
+
+# Check for the existence of elements in your AWS environment to see if it's necessary to create new versions of the same
+
+
+def check_and_set_context():
+    context_data = {}
+
+    # --- Find the VPC ID first ---
+    if VPC_NAME:
+        print("VPC_NAME:", VPC_NAME)
+        vpc_id, nat_gateways = get_vpc_id_by_name(VPC_NAME)
+
+        # If you expect only one, or one per AZ and you're creating one per AZ in CDK:
+        if nat_gateways:
+            # For simplicity, let's just check if *any* NAT exists in the VPC
+            # A more robust check would match by subnet, AZ, or a specific tag.
+            context_data["exists:NatGateway"] = True
+            context_data["id:NatGateway"] = nat_gateways[0][
+                "NatGatewayId"
+            ]  # Store the ID of the first one found
+        else:
+            context_data["exists:NatGateway"] = False
+            context_data["id:NatGateway"] = None
+
+        if not vpc_id:
+            # If the VPC doesn't exist, you might not be able to check/create subnets.
+            # Decide how to handle this: raise an error, set a flag, etc.
+            raise RuntimeError(
+                f"Required VPC '{VPC_NAME}' not found. Cannot proceed with subnet checks."
+            )
+
+        context_data["vpc_id"] = vpc_id  # Store VPC ID in context
+
+        # SUBNET CHECKS
+        all_proposed_subnets_data: List[Dict[str, str]] = []
+
+        # Flag to indicate if full validation mode (with CIDR/AZs) is active
+        full_validation_mode = False
+
+        # Determine if full validation mode is possible/desired
+        # It's 'desired' if CIDR/AZs are provided, and their lengths match the name lists.
+        public_ready_for_full_validation = (
+            len(PUBLIC_SUBNETS_TO_USE) > 0
+            and len(PUBLIC_SUBNET_CIDR_BLOCKS) == len(PUBLIC_SUBNETS_TO_USE)
+            and len(PUBLIC_SUBNET_AVAILABILITY_ZONES) == len(PUBLIC_SUBNETS_TO_USE)
+        )
+        private_ready_for_full_validation = (
+            len(PRIVATE_SUBNETS_TO_USE) > 0
+            and len(PRIVATE_SUBNET_CIDR_BLOCKS) == len(PRIVATE_SUBNETS_TO_USE)
+            and len(PRIVATE_SUBNET_AVAILABILITY_ZONES) == len(PRIVATE_SUBNETS_TO_USE)
+        )
+
+        # Activate full validation if *any* type of subnet (public or private) has its full details provided.
+        # You might adjust this logic if you require ALL subnet types to have CIDRs, or NONE.
+        if public_ready_for_full_validation or private_ready_for_full_validation:
+            full_validation_mode = True
+
+            # If some are ready but others aren't, print a warning or raise an error based on your strictness
+            if (
+                public_ready_for_full_validation
+                and not private_ready_for_full_validation
+                and PRIVATE_SUBNETS_TO_USE
+            ):
+                print(
+                    "Warning: Public subnets have CIDRs/AZs, but private subnets do not. Only public will be fully validated/created with CIDRs."
+                )
+            if (
+                private_ready_for_full_validation
+                and not public_ready_for_full_validation
+                and PUBLIC_SUBNETS_TO_USE
+            ):
+                print(
+                    "Warning: Private subnets have CIDRs/AZs, but public subnets do not. Only private will be fully validated/created with CIDRs."
+                )
+
+            # Prepare data for validate_subnet_creation_parameters for all subnets that have full details
+            if public_ready_for_full_validation:
+                for i, name in enumerate(PUBLIC_SUBNETS_TO_USE):
+                    all_proposed_subnets_data.append(
+                        {
+                            "name": name,
+                            "cidr": PUBLIC_SUBNET_CIDR_BLOCKS[i],
+                            "az": PUBLIC_SUBNET_AVAILABILITY_ZONES[i],
+                        }
+                    )
+            if private_ready_for_full_validation:
+                for i, name in enumerate(PRIVATE_SUBNETS_TO_USE):
+                    all_proposed_subnets_data.append(
+                        {
+                            "name": name,
+                            "cidr": PRIVATE_SUBNET_CIDR_BLOCKS[i],
+                            "az": PRIVATE_SUBNET_AVAILABILITY_ZONES[i],
+                        }
+                    )
+
+        print(f"Target VPC ID for Boto3 lookup: {vpc_id}")
+
+        # Fetch all existing subnets in the target VPC once to avoid repeated API calls
+        try:
+            existing_aws_subnets = _get_existing_subnets_in_vpc(vpc_id)
+        except Exception as e:
+            print(f"Failed to fetch existing VPC subnets. Aborting. Error: {e}")
+            raise SystemExit(1)  # Exit immediately if we can't get baseline data
+
+        print("\n--- Running Name-Only Subnet Existence Check Mode ---")
+        # Fallback: check only by name using the existing data
+        checked_public_subnets = {}
+        if PUBLIC_SUBNETS_TO_USE:
+            for subnet_name in PUBLIC_SUBNETS_TO_USE:
+                print("subnet_name:", subnet_name)
+                exists, subnet_id = check_subnet_exists_by_name(
+                    subnet_name, existing_aws_subnets
+                )
+                checked_public_subnets[subnet_name] = {
+                    "exists": exists,
+                    "id": subnet_id,
+                    "az": (
+                        existing_aws_subnets["by_name"].get(subnet_name, {}).get("az")
+                        if exists
+                        else None
+                    ),
+                    "route_table_id": (
+                        existing_aws_subnets["by_name"]
+                        .get(subnet_name, {})
+                        .get("route_table_id")
+                        if exists
+                        else None
+                    ),
+                }
+
+                # If the subnet exists, remove it from the proposed subnets list
+                if checked_public_subnets[subnet_name]["exists"] is True:
+                    all_proposed_subnets_data = [
+                        subnet
+                        for subnet in all_proposed_subnets_data
+                        if subnet["name"] != subnet_name
+                    ]
+
+        context_data["checked_public_subnets"] = checked_public_subnets
+
+        checked_private_subnets = {}
+        if PRIVATE_SUBNETS_TO_USE:
+            for subnet_name in PRIVATE_SUBNETS_TO_USE:
+                print("subnet_name:", subnet_name)
+                exists, subnet_id = check_subnet_exists_by_name(
+                    subnet_name, existing_aws_subnets
+                )
+                checked_private_subnets[subnet_name] = {
+                    "exists": exists,
+                    "id": subnet_id,
+                    "az": (
+                        existing_aws_subnets["by_name"].get(subnet_name, {}).get("az")
+                        if exists
+                        else None
+                    ),
+                    "route_table_id": (
+                        existing_aws_subnets["by_name"]
+                        .get(subnet_name, {})
+                        .get("route_table_id")
+                        if exists
+                        else None
+                    ),
+                }
+
+                # If the subnet exists, remove it from the proposed subnets list
+                if checked_private_subnets[subnet_name]["exists"] is True:
+                    all_proposed_subnets_data = [
+                        subnet
+                        for subnet in all_proposed_subnets_data
+                        if subnet["name"] != subnet_name
+                    ]
+
+        context_data["checked_private_subnets"] = checked_private_subnets
+
+        print("\nName-only existence subnet check complete.\n")
+
+        if full_validation_mode:
+            print(
+                "\n--- Running in Full Subnet Validation Mode (CIDR/AZs provided) ---"
+            )
+            try:
+                validate_subnet_creation_parameters(
+                    vpc_id, all_proposed_subnets_data, existing_aws_subnets
+                )
+                print("\nPre-synth validation successful. Proceeding with CDK synth.\n")
+
+                # Populate context_data for downstream CDK construct creation.
+                # Skip subnets that already exist in AWS (imported in the stack).
+                context_data["public_subnets_to_create"] = []
+                if public_ready_for_full_validation:
+                    for i, name in enumerate(PUBLIC_SUBNETS_TO_USE):
+                        if checked_public_subnets.get(name, {}).get("exists"):
+                            continue
+                        context_data["public_subnets_to_create"].append(
+                            {
+                                "name": name,
+                                "cidr": PUBLIC_SUBNET_CIDR_BLOCKS[i],
+                                "az": PUBLIC_SUBNET_AVAILABILITY_ZONES[i],
+                                "is_public": True,
+                            }
+                        )
+                context_data["private_subnets_to_create"] = []
+                if private_ready_for_full_validation:
+                    for i, name in enumerate(PRIVATE_SUBNETS_TO_USE):
+                        if checked_private_subnets.get(name, {}).get("exists"):
+                            continue
+                        context_data["private_subnets_to_create"].append(
+                            {
+                                "name": name,
+                                "cidr": PRIVATE_SUBNET_CIDR_BLOCKS[i],
+                                "az": PRIVATE_SUBNET_AVAILABILITY_ZONES[i],
+                                "is_public": False,
+                            }
+                        )
+
+            except (ValueError, Exception) as e:
+                print(f"\nFATAL ERROR: Subnet parameter validation failed: {e}\n")
+                raise SystemExit(1)  # Exit if validation fails
+
+    # Example checks and setting context values
+    # IAM Roles
+    role_name = CODEBUILD_ROLE_NAME
+    exists, role_arn, _ = check_for_existing_role(role_name)
+    context_data[f"exists:{role_name}"] = exists
+    if exists:
+        context_data[f"arn:{role_name}"] = role_arn
+
+    role_name = ECS_TASK_ROLE_NAME
+    exists, role_arn, _ = check_for_existing_role(role_name)
+    context_data[f"exists:{role_name}"] = exists
+    if exists:
+        context_data[f"arn:{role_name}"] = role_arn
+
+    role_name = ECS_TASK_EXECUTION_ROLE_NAME
+    exists, role_arn, _ = check_for_existing_role(role_name)
+    context_data[f"exists:{role_name}"] = exists
+    if exists:
+        context_data[f"arn:{role_name}"] = role_arn
+
+    # S3 Buckets
+    bucket_name = S3_LOG_CONFIG_BUCKET_NAME
+    exists, _ = check_s3_bucket_exists(bucket_name)
+    context_data[f"exists:{bucket_name}"] = exists
+    if exists:
+        # You might not need the ARN if using from_bucket_name
+        pass
+
+    output_bucket_name = S3_OUTPUT_BUCKET_NAME
+    exists, _ = check_s3_bucket_exists(output_bucket_name)
+    context_data[f"exists:{output_bucket_name}"] = exists
+    if exists:
+        pass
+
+    # ECR Repository
+    repo_name = ECR_CDK_REPO_NAME
+    exists, _ = check_ecr_repo_exists(repo_name)
+    context_data[f"exists:{repo_name}"] = exists
+    if exists:
+        pass  # from_repository_name is sufficient
+
+    # CodeBuild Project
+    project_name = CODEBUILD_PROJECT_NAME
+    exists, project_arn, service_role_arn = check_codebuild_project_exists(project_name)
+    context_data[f"exists:{project_name}"] = exists
+    if exists:
+        context_data[f"arn:{project_name}"] = project_arn
+        if service_role_arn:
+            context_data[f"service_role_arn:{project_name}"] = service_role_arn
+
+    # ALB (by name lookup) — context keys use the same 32-char name the stack uses
+    alb_name = ALB_NAME[-32:] if len(ALB_NAME) > 32 else ALB_NAME
+    exists, alb_object = check_alb_exists(alb_name, region_name=AWS_REGION)
+    context_data[f"exists:{alb_name}"] = exists
+    if exists:
+        print("alb_object:", alb_object)
+        context_data[f"arn:{alb_name}"] = alb_object["LoadBalancerArn"]
+        context_data[f"dns:{alb_name}"] = alb_object["DNSName"]
+        context_data[f"canonical_hosted_zone_id:{alb_name}"] = alb_object[
+            "CanonicalHostedZoneId"
+        ]
+        if alb_object.get("SecurityGroups"):
+            context_data[f"security_group_id:{alb_name}"] = alb_object[
+                "SecurityGroups"
+            ][0]
+
+    # Cognito User Pool (by name)
+    user_pool_name = COGNITO_USER_POOL_NAME
+    exists, user_pool_id, _ = check_for_existing_user_pool(user_pool_name)
+    context_data[f"exists:{user_pool_name}"] = exists
+    if exists:
+        context_data[f"id:{user_pool_name}"] = user_pool_id
+
+    # Cognito User Pool Client (by name and pool ID) - requires User Pool ID from check
+    if user_pool_id:
+        user_pool_id_for_client_check = user_pool_id  # context_data.get(f"id:{user_pool_name}") # Use ID from context
+        user_pool_client_name = COGNITO_USER_POOL_CLIENT_NAME
+        if user_pool_id_for_client_check:
+            exists, client_id, _ = check_for_existing_user_pool_client(
+                user_pool_client_name, user_pool_id_for_client_check
+            )
+            context_data[f"exists:{user_pool_client_name}"] = exists
+            if exists:
+                context_data[f"id:{user_pool_client_name}"] = client_id
+
+    # Secrets Manager Secret (by name)
+    secret_name = COGNITO_USER_POOL_CLIENT_SECRET_NAME
+    exists, _ = check_for_secret(secret_name)
+    context_data[f"exists:{secret_name}"] = exists
+    # You might not need the ARN if using from_secret_name_v2
+
+    # WAF Web ACL (by name and scope)
+    web_acl_name = WEB_ACL_NAME
+    exists, existing_web_acl = check_web_acl_exists(web_acl_name, scope="CLOUDFRONT")
+    context_data[f"exists:{web_acl_name}"] = exists
+    if exists:
+        context_data[f"arn:{web_acl_name}"] = existing_web_acl["ARN"]
+
+    # Write the context data to the file
+    with open(CONTEXT_FILE, "w") as f:
+        json.dump(context_data, f, indent=2)
+
+    print(f"Context data written to {CONTEXT_FILE}")

cdk/lambda_load_dynamo_logs.py

@@ -0,0 +1,321 @@
+"""
+Lambda handler to export DynamoDB usage log table to CSV and upload to S3.
+
+All inputs are read from environment variables (no argparse).
+Intended to run as an AWS Lambda function; can also be invoked locally
+by setting env vars and calling lambda_handler({}, None).
+
+Environment variables (same semantics as load_dynamo_logs.py CLI):
+  DYNAMODB_TABLE_NAME       - DynamoDB table name (default: redaction_usage)
+  AWS_REGION                - AWS region (optional; if unset, uses AWS_DEFAULT_REGION,
+                              then region from Lambda context ARN, then eu-west-2)
+  OUTPUT_FOLDER             - Local output directory, e.g. /tmp (optional)
+  OUTPUT_FILENAME           - Local output file name (default: dynamodb_logs_export.csv)
+  OUTPUT                    - Full local output path (overrides folder + filename if set).
+                              In Lambda only /tmp is writable; relative paths are auto-resolved to /tmp.
+  FROM_DATE                 - Only include entries on/after this date YYYY-MM-DD (optional)
+  TO_DATE                   - Only include entries on/before this date YYYY-MM-DD (optional)
+  DATE_ATTRIBUTE            - Attribute name for date filtering (default: timestamp)
+  S3_OUTPUT_BUCKET          - S3 bucket for the output CSV (required for upload)
+  S3_OUTPUT_KEY             - S3 object key/path for the output CSV (required for upload)
+"""
+
+import csv
+import datetime
+import os
+from decimal import Decimal
+from io import StringIO
+
+import boto3
+
+
+def _get_region_from_context(context):
+    """Extract region from Lambda context invoked_function_arn (arn:aws:lambda:REGION:ACCOUNT:function:NAME)."""
+    if context is None:
+        return None
+    arn = getattr(context, "invoked_function_arn", None)
+    if not arn or not isinstance(arn, str):
+        return None
+    parts = arn.split(":")
+    if len(parts) >= 4:
+        return parts[3]  # region is 4th segment
+    return None
+
+
+def get_config_from_env(context=None):
+    """Read all settings from environment variables (same inputs as load_dynamo_logs.py).
+    When running in Lambda, context can be passed to derive region from the function ARN if env is not set.
+    """
+    today = datetime.datetime.now().date()
+    one_year_ago = today - datetime.timedelta(days=365)
+
+    table_name = os.environ.get("DYNAMODB_TABLE_NAME") or os.environ.get(
+        "USAGE_LOG_DYNAMODB_TABLE_NAME", "redaction_usage"
+    )
+    region = (
+        os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or ""
+    ).strip()
+    output = os.environ.get("OUTPUT")
+    output_folder = os.environ.get("OUTPUT_FOLDER", "output/")
+    output_filename = os.environ.get("OUTPUT_FILENAME", "dynamodb_logs_export.csv")
+    from_date_str = os.environ.get("FROM_DATE")
+    to_date_str = os.environ.get("TO_DATE")
+    date_attribute = os.environ.get("DATE_ATTRIBUTE", "timestamp")
+    s3_output_bucket = os.environ.get("S3_OUTPUT_BUCKET")
+    s3_output_key = os.environ.get("S3_OUTPUT_KEY")
+
+    if output:
+        local_output_path = output
+    else:
+        folder = output_folder.rstrip("/").rstrip("\\")
+        local_output_path = os.path.join(folder, output_filename)
+
+    # In AWS Lambda only /tmp is writable; resolve relative paths to /tmp to avoid read-only FS errors
+    if os.environ.get("AWS_LAMBDA_FUNCTION_NAME"):
+        resolved = os.path.abspath(local_output_path)
+        if not resolved.startswith("/tmp"):
+            local_output_path = os.path.join(
+                "/tmp", os.path.basename(local_output_path)
+            )
+
+    # Region: env (AWS_REGION / AWS_DEFAULT_REGION) → Lambda context ARN → hardcoded fallback
+    if not region and context is not None:
+        region = _get_region_from_context(context) or ""
+    if not region:
+        region = "FILL IN DEFAULT REGION HERE"
+
+    from_date = None
+    to_date = None
+    if from_date_str:
+        from_date = datetime.datetime.strptime(from_date_str, "%Y-%m-%d").date()
+    if to_date_str:
+        to_date = datetime.datetime.strptime(to_date_str, "%Y-%m-%d").date()
+    if from_date is None and to_date is None:
+        from_date = one_year_ago
+        to_date = today
+    elif from_date is None:
+        from_date = one_year_ago
+    elif to_date is None:
+        to_date = today
+
+    return {
+        "table_name": table_name,
+        "region": region,
+        "local_output_path": local_output_path,
+        "from_date": from_date,
+        "to_date": to_date,
+        "date_attribute": date_attribute,
+        "s3_output_bucket": s3_output_bucket,
+        "s3_output_key": s3_output_key,
+    }
+
+
+# Helper function to convert Decimal to float or int
+def convert_types(item):
+    new_item = {}
+    for key, value in item.items():
+        if isinstance(value, Decimal):
+            new_item[key] = int(value) if value % 1 == 0 else float(value)
+        elif isinstance(value, str):
+            try:
+                dt_obj = datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
+                new_item[key] = dt_obj.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+            except (ValueError, TypeError):
+                new_item[key] = value
+        else:
+            new_item[key] = value
+    return new_item
+
+
+def _parse_item_date(value):
+    """Parse a DynamoDB attribute value to datetime for comparison. Returns None if unparseable."""
+    if value is None:
+        return None
+    if isinstance(value, Decimal):
+        try:
+            return datetime.datetime.utcfromtimestamp(float(value))
+        except (ValueError, OSError):
+            return None
+    if isinstance(value, (int, float)):
+        try:
+            return datetime.datetime.utcfromtimestamp(float(value))
+        except (ValueError, OSError):
+            return None
+    if isinstance(value, str):
+        for fmt in (
+            "%Y-%m-%d %H:%M:%S.%f",
+            "%Y-%m-%d %H:%M:%S",
+            "%Y-%m-%d",
+            "%Y-%m-%dT%H:%M:%S",
+        ):
+            try:
+                return datetime.datetime.strptime(value, fmt)
+            except (ValueError, TypeError):
+                continue
+        try:
+            return datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except (ValueError, TypeError):
+            pass
+    return None
+
+
+def filter_items_by_date(items, from_date, to_date, date_attribute: str):
+    """Return items whose date attribute falls within [from_date, to_date] (inclusive)."""
+    if from_date is None and to_date is None:
+        return items
+    start = datetime.datetime.combine(from_date, datetime.time.min)
+    end = datetime.datetime.combine(to_date, datetime.time.max)
+    filtered = []
+    for item in items:
+        raw = item.get(date_attribute)
+        dt = _parse_item_date(raw)
+        if dt is None:
+            continue
+        if dt.tzinfo:
+            dt = dt.replace(tzinfo=None)
+        if start <= dt <= end:
+            filtered.append(item)
+    return filtered
+
+
+def scan_table(table):
+    """Paginated scan of DynamoDB table."""
+    items = []
+    response = table.scan()
+    items.extend(response["Items"])
+    while "LastEvaluatedKey" in response:
+        response = table.scan(ExclusiveStartKey=response["LastEvaluatedKey"])
+        items.extend(response["Items"])
+    return items
+
+
+def export_to_csv_buffer(items, fields_to_drop=None):
+    """
+    Write items to a CSV in memory; return (csv_string, fieldnames).
+    Use for uploading to S3 without writing to disk.
+    """
+    if not items:
+        return "", []
+
+    drop_set = set(fields_to_drop or [])
+    all_keys = set()
+    for item in items:
+        all_keys.update(item.keys())
+    fieldnames = sorted(list(all_keys - drop_set))
+
+    buf = StringIO()
+    writer = csv.DictWriter(
+        buf, fieldnames=fieldnames, extrasaction="ignore", restval=""
+    )
+    writer.writeheader()
+    for item in items:
+        writer.writerow(convert_types(item))
+    return buf.getvalue(), fieldnames
+
+
+def export_to_csv_file(items, output_path, fields_to_drop=None):
+    """Write items to a CSV file (for optional /tmp or local path)."""
+    csv_string, _ = export_to_csv_buffer(items, fields_to_drop)
+    if not csv_string:
+        return
+    os.makedirs(os.path.dirname(os.path.abspath(output_path)) or ".", exist_ok=True)
+    with open(output_path, "w", newline="", encoding="utf-8-sig") as f:
+        f.write(csv_string)
+
+
+def run_export(config):
+    """
+    Run the full export: scan DynamoDB, filter by date, write CSV (buffer and/or file), upload to S3.
+    """
+    table_name = config["table_name"]
+    region = config["region"]
+    local_output_path = config["local_output_path"]
+    from_date = config["from_date"]
+    to_date = config["to_date"]
+    date_attribute = config["date_attribute"]
+    s3_output_bucket = config["s3_output_bucket"]
+    s3_output_key = config["s3_output_key"]
+
+    if from_date > to_date:
+        raise ValueError("FROM_DATE must be on or before TO_DATE")
+
+    dynamodb = boto3.resource("dynamodb", region_name=region or None)
+    table = dynamodb.Table(table_name)
+
+    items = scan_table(table)
+    items = filter_items_by_date(items, from_date, to_date, date_attribute)
+
+    csv_string, fieldnames = export_to_csv_buffer(items, fields_to_drop=[])
+    result = {
+        "item_count": len(items),
+        "from_date": str(from_date),
+        "to_date": str(to_date),
+        "columns": fieldnames,
+    }
+
+    if csv_string:
+        # Optional: write to local path (e.g. /tmp in Lambda)
+        try:
+            export_to_csv_file(items, local_output_path, fields_to_drop=[])
+            result["local_path"] = local_output_path
+        except Exception as e:
+            result["local_write_error"] = str(e)
+
+        # Upload to S3 if bucket and key are set
+        if s3_output_bucket and s3_output_key:
+            s3 = boto3.client("s3", region_name=region or None)
+            s3.put_object(
+                Bucket=s3_output_bucket,
+                Key=s3_output_key,
+                Body=csv_string.encode("utf-8-sig"),
+                ContentType="text/csv; charset=utf-8",
+            )
+            result["s3_uri"] = f"s3://{s3_output_bucket}/{s3_output_key}"
+        elif s3_output_bucket or s3_output_key:
+            result["s3_skip_reason"] = (
+                "Both S3_OUTPUT_BUCKET and S3_OUTPUT_KEY must be set"
+            )
+
+    return result
+
+
+def lambda_handler(event, context):
+    """
+    AWS Lambda entrypoint. Config is read from environment variables.
+
+    Event is not required for config; it can be used to override env vars
+    (e.g. pass table_name, from_date, to_date, s3_output_bucket, s3_output_key).
+    """
+    config = get_config_from_env(context=context)
+
+    # Optional: allow event to override env-based config
+    if isinstance(event, dict):
+        if event.get("table_name"):
+            config["table_name"] = event["table_name"]
+        if event.get("region"):
+            config["region"] = event["region"]
+        if event.get("from_date"):
+            config["from_date"] = datetime.datetime.strptime(
+                event["from_date"], "%Y-%m-%d"
+            ).date()
+        if event.get("to_date"):
+            config["to_date"] = datetime.datetime.strptime(
+                event["to_date"], "%Y-%m-%d"
+            ).date()
+        if event.get("date_attribute"):
+            config["date_attribute"] = event["date_attribute"]
+        if event.get("s3_output_bucket"):
+            config["s3_output_bucket"] = event["s3_output_bucket"]
+        if event.get("s3_output_key"):
+            config["s3_output_key"] = event["s3_output_key"]
+
+    result = run_export(config)
+    return {"statusCode": 200, "body": result}
+
+
+if __name__ == "__main__":
+    # Allow running locally with env vars set
+    import json
+
+    result = lambda_handler({}, None)
+    print(json.dumps(result, indent=2))

cdk/post_cdk_build_quickstart.py

@@ -0,0 +1,40 @@
+import time
+
+from cdk_config import (
+    CLUSTER_NAME,
+    CODEBUILD_PROJECT_NAME,
+    ECS_SERVICE_NAME,
+    S3_LOG_CONFIG_BUCKET_NAME,
+)
+from cdk_functions import (
+    create_basic_config_env,
+    start_codebuild_build,
+    start_ecs_task,
+    upload_file_to_s3,
+)
+from tqdm import tqdm
+
+# Create basic config.env file that user can use to run the app later. Input is the folder it is saved into.
+create_basic_config_env("config")
+
+# Start codebuild build
+print("Starting CodeBuild project.")
+start_codebuild_build(PROJECT_NAME=CODEBUILD_PROJECT_NAME)
+
+# Upload config.env file to S3 bucket
+upload_file_to_s3(
+    local_file_paths="config/config.env", s3_key="", s3_bucket=S3_LOG_CONFIG_BUCKET_NAME
+)
+
+total_seconds = 660  # 11 minutes
+update_interval = 1  # Update every second
+
+print("Waiting 11 minutes for the CodeBuild container to build.")
+
+# tqdm iterates over a range, and you perform a small sleep in each iteration
+for i in tqdm(range(total_seconds), desc="Building container"):
+    time.sleep(update_interval)
+
+# Start task on ECS
+print("Starting ECS task")
+start_ecs_task(cluster_name=CLUSTER_NAME, service_name=ECS_SERVICE_NAME)

cdk/requirements.txt

@@ -0,0 +1,5 @@
+aws-cdk-lib==2.257.0
+boto3<=1.42.91
+pandas<=2.3.3
+nodejs<=0.1.1
+python-dotenv<=1.2.2

doc_redaction/__init__.py

@@ -0,0 +1,27 @@
+"""
+doc_redaction package.
+
+This package layer is intentionally thin for now: it preserves existing
+repo-root entrypoints (e.g. `app.py`, `cli_redact.py`) while providing stable
+import paths for PyPI installs.
+"""
+
+from __future__ import annotations
+
+__all__ = ["__version__", "choose_and_run_redactor", "run_redaction"]
+
+try:
+    from importlib.metadata import PackageNotFoundError, version
+
+    try:
+        __version__ = version("doc_redaction")
+    except PackageNotFoundError:  # pragma: no cover
+        __version__ = "0.0.0"
+except Exception:  # pragma: no cover
+    __version__ = "0.0.0"
+
+# Convenience re-exports (package-qualified import surface)
+from doc_redaction.file_redaction import (
+    choose_and_run_redactor,
+    run_redaction,
+)  # noqa: E402

doc_redaction/api.py

@@ -0,0 +1,43 @@
+"""
+Stable programmatic API surface matching Gradio `api_name` values.
+
+This module provides names that exactly match the Gradio endpoint `api_name`
+strings from `app.py`.
+
+By default these names point to the **CLI-first** Python API (`doc_redaction.cli_api`),
+which is the most stable and runnable interface outside Gradio session state.
+"""
+
+from __future__ import annotations
+
+from doc_redaction.cli_api import (
+    apply_review_redactions,
+    combine_review_csvs,
+    combine_review_pdfs,
+    export_review_page_ocr_visualisation,
+    export_review_redaction_overlay,
+    find_duplicate_pages,
+    find_duplicate_tabular,
+    load_and_prepare_documents_or_data,
+    redact_data,
+    redact_document,
+    summarise_document,
+    verify_redaction_coverage,
+    word_level_ocr_text_search,
+)
+
+__all__ = [
+    "redact_document",
+    "load_and_prepare_documents_or_data",
+    "apply_review_redactions",
+    "export_review_page_ocr_visualisation",
+    "export_review_redaction_overlay",
+    "verify_redaction_coverage",
+    "word_level_ocr_text_search",
+    "redact_data",
+    "find_duplicate_pages",
+    "find_duplicate_tabular",
+    "summarise_document",
+    "combine_review_csvs",
+    "combine_review_pdfs",
+]

doc_redaction/cli_api.py

@@ -0,0 +1,405 @@
+"""
+CLI-first programmatic API surface.
+
+These functions provide a minimal, runnable Python interface that mirrors the
+Gradio `api_name` routes, but executes the underlying workflows via the CLI
+engine (`cli_redact.main(direct_mode_args=...)`).
+
+Return values are lists of output file paths created in `output_dir`.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from pathlib import Path
+from typing import Any, Iterable
+
+
+def _ensure_list(v: str | list[str] | tuple[str, ...]) -> list[str]:
+    if isinstance(v, (list, tuple)):
+        return [str(x) for x in v]
+    return [str(v)]
+
+
+def _snapshot_files(folder: str) -> set[str]:
+    root = Path(folder)
+    if not root.exists():
+        return set()
+    out: set[str] = set()
+    for dirpath, _, filenames in os.walk(root):
+        for name in filenames:
+            out.add(str(Path(dirpath) / name))
+    return out
+
+
+def _default_output_dir(prefix: str) -> str:
+    return tempfile.mkdtemp(prefix=f"doc_redaction_{prefix}_")
+
+
+def _run_cli(
+    *,
+    gradio_api_name: str,
+    overrides: dict[str, Any],
+    output_dir: str | None,
+) -> list[str]:
+    """
+    Run cli_redact.main with merged defaults and return newly created files.
+    """
+    from cli_redact import get_cli_default_args_dict
+    from cli_redact import main as cli_main
+
+    merged = get_cli_default_args_dict()
+    merged.update(overrides)
+
+    if output_dir is None:
+        output_dir = _default_output_dir(gradio_api_name)
+    merged["output_dir"] = str(output_dir)
+
+    before = _snapshot_files(str(output_dir))
+    cli_main(direct_mode_args=merged)
+    after = _snapshot_files(str(output_dir))
+
+    created = sorted(after - before)
+    return created
+
+
+# ---------------------------------------------------------------------------
+# Implemented via CLI engine (matches agent_routes.py)
+# ---------------------------------------------------------------------------
+
+
+def redact_document(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    ocr_method: str | None = None,
+    pii_detector: str | None = None,
+    instruction: str | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """
+    Parity with Gradio `api_name='redact_document'`.
+    Runs CLI task `redact` (PDF/PNG/JPG) or relevant workflow based on file type.
+    """
+    direct: dict[str, Any] = {
+        "task": "redact",
+        "input_file": _ensure_list(input_files),
+    }
+    if ocr_method is not None:
+        direct["ocr_method"] = ocr_method
+    if pii_detector is not None:
+        direct["pii_detector"] = pii_detector
+    if instruction is not None:
+        direct["custom_llm_instructions"] = instruction
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="redact_document", overrides=direct, output_dir=output_dir
+    )
+
+
+def redact_data(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    instruction: str | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='redact_data'` (same CLI task: `redact`)."""
+    direct: dict[str, Any] = {"task": "redact", "input_file": _ensure_list(input_files)}
+    if instruction is not None:
+        direct["custom_llm_instructions"] = instruction
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="redact_data", overrides=direct, output_dir=output_dir
+    )
+
+
+def find_duplicate_pages(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    similarity_threshold: float | None = None,
+    min_word_count: int | None = None,
+    min_consecutive_pages: int | None = None,
+    greedy_match: bool | None = None,
+    combine_pages: bool | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='find_duplicate_pages'`."""
+    direct: dict[str, Any] = {
+        "task": "deduplicate",
+        "duplicate_type": "pages",
+        "input_file": _ensure_list(input_files),
+    }
+    if similarity_threshold is not None:
+        direct["similarity_threshold"] = similarity_threshold
+    if min_word_count is not None:
+        direct["min_word_count"] = min_word_count
+    if min_consecutive_pages is not None:
+        direct["min_consecutive_pages"] = min_consecutive_pages
+    if greedy_match is not None:
+        direct["greedy_match"] = "True" if greedy_match else "False"
+    if combine_pages is not None:
+        direct["combine_pages"] = "True" if combine_pages else "False"
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="find_duplicate_pages", overrides=direct, output_dir=output_dir
+    )
+
+
+def find_duplicate_tabular(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    text_columns: list[str] | None = None,
+    similarity_threshold: float | None = None,
+    min_word_count: int | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='find_duplicate_tabular'`."""
+    direct: dict[str, Any] = {
+        "task": "deduplicate",
+        "duplicate_type": "tabular",
+        "input_file": _ensure_list(input_files),
+    }
+    if text_columns is not None:
+        direct["text_columns"] = list(text_columns)
+    if similarity_threshold is not None:
+        direct["similarity_threshold"] = similarity_threshold
+    if min_word_count is not None:
+        direct["min_word_count"] = min_word_count
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="find_duplicate_tabular",
+        overrides=direct,
+        output_dir=output_dir,
+    )
+
+
+def summarise_document(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='summarise_document'` (CLI task: `summarise`)."""
+    direct: dict[str, Any] = {
+        "task": "summarise",
+        "input_file": _ensure_list(input_files),
+    }
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="summarise_document", overrides=direct, output_dir=output_dir
+    )
+
+
+def combine_review_pdfs(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='combine_review_pdfs'` (CLI task: `combine_review_pdfs`)."""
+    direct: dict[str, Any] = {
+        "task": "combine_review_pdfs",
+        "input_file": _ensure_list(input_files),
+    }
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="combine_review_pdfs", overrides=direct, output_dir=output_dir
+    )
+
+
+# ---------------------------------------------------------------------------
+# Implemented without CLI (as per agent_routes.py)
+# ---------------------------------------------------------------------------
+
+
+def combine_review_csvs(
+    input_files: Iterable[str],
+    *,
+    output_dir: str | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='combine_review_csvs'`."""
+    from tools.config import OUTPUT_FOLDER
+    from tools.helper_functions import merge_csv_files
+
+    out_dir = str(output_dir or OUTPUT_FOLDER)
+    Path(out_dir).mkdir(parents=True, exist_ok=True)
+    sep = "/" if not out_dir.endswith(("/", "\\")) else ""
+
+    return merge_csv_files([str(p) for p in input_files], output_folder=out_dir + sep)
+
+
+def export_review_redaction_overlay(
+    *,
+    page_image_path: str,
+    boxes: list[dict[str, Any]],
+    page_number: int = 1,
+    doc_base_name: str = "review",
+    review_df_records: list[dict[str, Any]] | None = None,
+    label_abbrev_chars: int | None = None,
+) -> list[str]:
+    """Same behaviour as Gradio ``api_name='page_redaction_review_image'``; Agent API route ``export_review_redaction_overlay``."""
+    import pandas as pd
+
+    from tools.config import OUTPUT_FOLDER
+    from tools.redaction_review import visualise_review_redaction_boxes
+
+    annotator: dict[str, Any] = {"image": page_image_path, "boxes": boxes}
+    review_df = pd.DataFrame(review_df_records) if review_df_records else pd.DataFrame()
+
+    out_dir = str(Path(OUTPUT_FOLDER).expanduser().resolve())
+    Path(out_dir).mkdir(parents=True, exist_ok=True)
+    out_path = visualise_review_redaction_boxes(
+        annotator,
+        review_df=review_df,
+        output_folder=out_dir,
+        page_number=page_number,
+        doc_base_name=doc_base_name,
+        label_abbrev_chars=label_abbrev_chars,
+    )
+    return [out_path] if out_path else []
+
+
+def export_review_page_ocr_visualisation(
+    *,
+    page_image_path: str,
+    ocr_results: dict[str, Any],
+    page_number: int = 1,
+    doc_base_name: str = "review",
+) -> list[str]:
+    """Same behaviour as Gradio ``api_name='page_ocr_review_image'``; Agent API route ``export_review_page_ocr_visualisation``."""
+    from PIL import Image
+
+    from tools.config import OUTPUT_FOLDER
+    from tools.file_redaction import visualise_ocr_words_bounding_boxes
+
+    out_dir = str(Path(OUTPUT_FOLDER).expanduser().resolve())
+    Path(out_dir).mkdir(parents=True, exist_ok=True)
+
+    image_name = f"{str(doc_base_name or 'review')}_page{int(page_number)}.png"
+    log_paths: list[str] = []
+    log_paths = visualise_ocr_words_bounding_boxes(
+        Image.open(page_image_path).convert("RGB"),
+        ocr_results,
+        image_name=image_name,
+        output_folder=out_dir,
+        visualisation_folder="review_ocr_visualisations",
+        add_legend=True,
+        log_files_output_paths=log_paths,
+    )
+    return list(log_paths)
+
+
+# ---------------------------------------------------------------------------
+# Gradio-session-only (no single CLI task)
+# ---------------------------------------------------------------------------
+
+
+def load_and_prepare_documents_or_data(*args: Any, **kwargs: Any) -> list[str]:
+    raise NotImplementedError(
+        "load_and_prepare_documents_or_data is Gradio-session-state driven and is not exposed as a single CLI task."
+    )
+
+
+def apply_review_redactions(
+    pdf_path: str,
+    review_csv_path: str,
+    *,
+    output_dir: str | None = None,
+    input_dir: str | None = None,
+    text_extract_method: str | None = None,
+    efficient_ocr: bool | None = None,
+) -> list[str]:
+    """
+    Headless parity with Gradio ``api_name='apply_review_redactions'``.
+
+    Returns output file paths (redacted PDF, review CSV, logs, etc.).
+    """
+    from tools.simplified_api import run_apply_review_redactions
+
+    r = run_apply_review_redactions(
+        pdf_path=pdf_path,
+        review_csv_path=review_csv_path,
+        output_dir=output_dir,
+        input_dir=input_dir,
+        text_extract_method=text_extract_method,
+        efficient_ocr=efficient_ocr,
+    )
+    return list(r.get("output_paths") or [])
+
+
+def word_level_ocr_text_search(
+    ocr_words_csv_path: str,
+    search_text: str,
+    *,
+    similarity_threshold: float = 1.0,
+    use_regex: bool = False,
+    review_csv_path: str | None = None,
+) -> dict:
+    """Headless word-level OCR search against ``*_ocr_results_with_words_*.csv``."""
+    from tools.verify_redaction_coverage import run_word_level_ocr_text_search
+
+    return run_word_level_ocr_text_search(
+        ocr_words_csv_path,
+        search_text,
+        similarity_threshold=similarity_threshold,
+        use_regex=use_regex,
+        review_csv_path=review_csv_path,
+    )
+
+
+def verify_redaction_coverage(
+    review_csv_path: str,
+    ocr_words_csv_path: str,
+    *,
+    must_redact: list[str] | None = None,
+    must_not_redact: list[str] | None = None,
+    redacted_pdf_path: str | None = None,
+    total_pages: int | None = None,
+    min_word_length: int = 3,
+    sample_pixels: bool = False,
+    auto_prune_suspicious: bool = False,
+    pruned_output_path: str | None = None,
+) -> dict:
+    """Pass 1 programmatic coverage report (no VLM)."""
+    from tools.simplified_api import run_verify_redaction_coverage
+
+    report, _, _ = run_verify_redaction_coverage(
+        review_csv_path,
+        ocr_words_csv_path,
+        must_redact=must_redact,
+        must_not_redact=must_not_redact,
+        redacted_pdf_path=redacted_pdf_path,
+        total_pages=total_pages,
+        min_word_length=min_word_length,
+        sample_pixels=sample_pixels,
+        auto_prune_suspicious=auto_prune_suspicious,
+        pruned_output_path=pruned_output_path,
+    )
+    return report
+
+
+__all__ = [
+    "redact_document",
+    "load_and_prepare_documents_or_data",
+    "apply_review_redactions",
+    "export_review_page_ocr_visualisation",
+    "export_review_redaction_overlay",
+    "word_level_ocr_text_search",
+    "verify_redaction_coverage",
+    "redact_data",
+    "find_duplicate_pages",
+    "find_duplicate_tabular",
+    "summarise_document",
+    "combine_review_csvs",
+    "combine_review_pdfs",
+]

doc_redaction/cli_redact.py

@@ -0,0 +1,26 @@
+"""
+CLI entrypoint for packaging.
+
+Re-exports the existing repo-root `cli_redact.py` implementation so that
+`pyproject.toml` console scripts can target a stable package path.
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any, Dict
+
+_root_cli = importlib.import_module("cli_redact")
+
+build_cli_argument_parser = getattr(_root_cli, "build_cli_argument_parser")
+get_cli_default_args_dict = getattr(_root_cli, "get_cli_default_args_dict")
+
+
+def main(direct_mode_args: Dict[str, Any] | None = None):
+    # Mirror the root signature but avoid a mutable default.
+    if direct_mode_args is None:
+        direct_mode_args = {}
+    return _root_cli.main(direct_mode_args=direct_mode_args)
+
+
+__all__ = ["build_cli_argument_parser", "get_cli_default_args_dict", "main"]

doc_redaction/data_anonymise.py

@@ -0,0 +1,9 @@
+"""
+Public API wrappers for tabular anonymisation functions.
+"""
+
+from __future__ import annotations
+
+from tools.data_anonymise import anonymise_files_with_open_text
+
+__all__ = ["anonymise_files_with_open_text"]

doc_redaction/example_data/Bold minimalist professional cover letter.docx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c8551ac157f350b2093e5d8c89f68474f613350074201cff6d52d5ed5ec28ff
+size 23992

doc_redaction/example_data/Example-cv-university-graduaty-hr-role-with-photo-2.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:caf00ca5cb06b8019804d1a7eaeceec772607969e8cad6c34d1d583876345b90
+size 116763

doc_redaction/example_data/Partnership-Agreement-Toolkit_0_0.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0db46a784d7aaafb8d02acf8686523dd376400117d07926a5dcb51ceb69e3236
+size 426602

doc_redaction/example_data/Partnership-Agreement-Toolkit_test_deny_list_para_single_spell.csv

@@ -0,0 +1,2 @@
+another  country or territory sign a formel agreement on behalf? of their communities endorsing a
+soster citues international

doc_redaction/example_data/combined_case_notes.csv

@@ -0,0 +1,19 @@
+Date,Social Worker,Client,Case Note
+"January 3, 2023",Jane Smith,Alex D.,"Met with Alex at school following reports of increased absences and declining grades. Alex appeared sullen and avoided eye contact. When prompted about school, Alex expressed feelings of isolation and stated, ""No one gets me."" Scheduled a follow-up meeting to further explore these feelings."
+"January 17, 2023",Jane Smith,Alex D.,"Met with Alex at the community center. Alex displayed sudden outbursts of anger when discussing home life, particularly in relation to a new stepfather. Alex mentioned occasional substance use, but did not specify which substances. Recommended a comprehensive assessment."
+"February 5, 2023",Jane Smith,Alex D.,Home visit conducted. Alex's mother reported frequent arguments at home. She expressed concerns about Alex's new group of friends and late-night outings. Noted potential signs of substance abuse. Suggested family counseling.
+"February 21, 2023",Jane Smith,Alex D.,"Met with Alex alone at my office. Alex appeared more agitated than in previous meetings. There were visible signs of self-harm on Alex's arms. When questioned, Alex became defensive. Immediate referral made to a mental health professional."
+"March 10, 2023",Jane Smith,Alex D.,Attended joint session with Alex and a therapist. Alex shared feelings of hopelessness and admitted to occasional thoughts of self-harm. Therapist recommended a comprehensive mental health evaluation and ongoing therapy.
+"March 25, 2023",Jane Smith,Alex D.,"Received a call from Alex's school about a physical altercation with another student. Met with Alex, who displayed high levels of frustration and admitted to the use of alcohol. Discussed the importance of seeking help and finding positive coping mechanisms. Recommended enrollment in an anger management program."
+"April 15, 2023",Jane Smith,Alex D.,Met with Alex and mother to discuss progress. Alex's mother expressed concerns about Alex's increasing aggression at home. Alex acknowledged the issues but blamed others for provoking the behavior. It was decided that a more intensive intervention may be needed.
+"April 30, 2023",Jane Smith,Alex D.,"Met with Alex and a psychiatrist. Psychiatrist diagnosed Alex with Oppositional Defiant Disorder (ODD) and co-morbid substance use disorder. A treatment plan was discussed, including medication, therapy, and family counseling."
+"May 20, 2023",Jane Smith,Alex D.,"Met with Alex to discuss progress. Alex has started attending group therapy and has shown slight improvements in behavior. Still, concerns remain about substance use. Discussed potential for a short-term residential treatment program."
+"January 3, 2023",Jane Smith,Jamie L.,"Met with Jamie at school after receiving reports of consistent tardiness and decreased participation in class. Jamie appeared withdrawn and exhibited signs of sadness. When asked about feelings, Jamie expressed feeling ""empty"" and ""hopeless"" at times. Scheduled a follow-up meeting to further explore these feelings."
+"January 17, 2023",Jane Smith,Jamie L.,"Met with Jamie at the community center. Jamie shared feelings of low self-worth, mentioning that it's hard to find motivation for daily tasks. Discussed potential triggers and learned about recent family financial struggles. Recommended counseling and possible group therapy for peer support."
+"February 5, 2023",Jane Smith,Jamie L.,Home visit conducted. Jamie's parents shared concerns about Jamie's increasing withdrawal from family activities and lack of interest in hobbies. Parents mentioned that Jamie spends a lot of time alone in the room. Suggested family therapy to open communication channels.
+"February 21, 2023",Jane Smith,Jamie L.,Met with Jamie in my office. Jamie opened up about feelings of isolation and mentioned difficulty sleeping. No signs of self-harm or suicidal ideation were noted. Recommended a comprehensive mental health assessment to better understand the depth of the depression.
+"March 10, 2023",Jane Smith,Jamie L.,"Attended a joint session with Jamie and a therapist. The therapist noted signs of moderate depression. Together, we discussed coping strategies and potential interventions. Jamie showed interest in art therapy."
+"March 25, 2023",Jane Smith,Jamie L.,"Received feedback from Jamie's school that academic performance has slightly improved. However, social interactions remain limited. Encouraged Jamie to join school clubs or groups to foster connection."
+"April 15, 2023",Jane Smith,Jamie L.,"Met with Jamie and parents to discuss progress. Parents have observed slight improvements in mood on some days, but overall, Jamie still appears to struggle. It was decided to explore medication as a potential aid alongside therapy."
+"April 30, 2023",Jane Smith,Jamie L.,Met with Jamie and a psychiatrist. The psychiatrist diagnosed Jamie with Major Depressive Disorder (MDD) and suggested considering antidepressant medication. Discussed the potential benefits and side effects. Jamie and parents will think it over.
+"May 20, 2023",Jane Smith,Jamie L.,"Jamie has started on a low dose of an antidepressant. Initial feedback is positive, with some improvement in mood and energy levels. Will continue monitoring and adjusting as necessary."

doc_redaction/example_data/combined_case_notes.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09300597024591d0b5b4ef97faef12fcceb28fcbb6ea09260bc42f43967753a4
+size 12579

doc_redaction/example_data/doubled_output_joined.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6eeac353164447c2aa429196e1a6ffae4c095d7171e63c2d1cd1966fdf32d1ed
+size 1274719

doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed0cd82b5b5826b851ca0e7c102d2d4d27580f7a90de4211a33178a6664d008d
+size 8848

doc_redaction/example_data/example_outputs/Partnership-Agreement-Toolkit_0_0.pdf_ocr_output.csv

@@ -0,0 +1,277 @@
+page,text,left,top,width,height,line
+1,Partnership Agreement,0.516078,0.027879,0.440784,0.032424,1
+1,SisterCities,0.169804,0.033333,0.238431,0.028182,2
+1,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788,3
+1,Toolkit,0.830588,0.07303,0.126667,0.025152,4
+1,Connect globally. Thrive locally.,0.169804,0.08697,0.238824,0.01303,5
+1,Types of Affiliations,0.117255,0.157576,0.241961,0.02,6
+1,Sister City Relationship,0.117647,0.187273,0.196863,0.013939,7
+1,"A Sister City relationship is formed when the mayor or highest elected official (or, if elections",0.117255,0.211212,0.738824,0.013636,8
+1,"do not take place, highest appointed official) from a U.S. community and a community in",0.117647,0.227273,0.70902,0.013939,9
+1,another country or territory sign a formal agreement on behalf of their communities endorsing a,0.117647,0.243636,0.761961,0.013636,10
+1,"""sister city/sister cities"" relationship. Sister city agreements shall be considered active/valid",0.118039,0.259697,0.731373,0.013939,11
+1,unless otherwise indicated by one or both of the respective communities.,0.118039,0.276061,0.58549,0.013636,12
+1,Sister Cities International shall formally recognize only those relationships by cities/members in,0.118039,0.299697,0.758824,0.013636,13
+1,good standing (i.e. who are current on membership dues) in its Membership Directory or on its,0.117647,0.316061,0.754902,0.013636,14
+1,"website. However, Sister Cities International shall not assert as invalid or otherwise impugn the",0.116863,0.332121,0.760784,0.013636,15
+1,legitimacy of those relationships formed by non-members.,0.118039,0.348485,0.466275,0.013636,16
+1,Friendship City,0.118039,0.372121,0.127059,0.013939,17
+1,"A Friendship City or Friendship Cities relationship is often formed by cities as a ""stepping",0.117255,0.395758,0.714118,0.013636,18
+1,"stone"" to a more formal ""Sister City"" agreement. Typically Friendship City agreements are",0.117647,0.411515,0.720392,0.014242,19
+1,referred to as such in the formal documents that are signed. Sister Cities International shall,0.118039,0.428182,0.72549,0.013636,20
+1,recognize Friendship City relationships by members in its Membership Directory and website.,0.118039,0.444242,0.747843,0.013636,21
+1,As per Sister Cities International Board of Directors:,0.117255,0.467879,0.413333,0.013636,22
+1,Sister Cities International will recognize a new sister cities affiliation between a,0.169412,0.492121,0.626667,0.013333,23
+1,"U.S. and an international community, even though another affiliation may exist",0.169412,0.507879,0.625098,0.013636,24
+1,"between that international community and a different U.S. community, only if a",0.169412,0.524545,0.62902,0.013636,25
+1,cooperative agreement among all involved communities is filed with Sister Cities,0.16902,0.540606,0.643137,0.013636,26
+1,"International. If a cooperative agreement is denied, or no response to the request",0.170196,0.556667,0.647843,0.013333,27
+1,"is received within a reasonable amount of time, Sister Cities International will",0.169412,0.57303,0.612157,0.012727,28
+1,recognize the partnership as a friendship city and it will be delineated as such,0.169412,0.589091,0.621176,0.013636,29
+1,with a symbol in the membership directories.,0.168627,0.605455,0.358824,0.013333,30
+1,The cooperative agreement must be sent by the Mayor/County,0.168627,0.628788,0.509412,0.013939,31
+1,"Executive/Governor of the requesting community, and must be sent to the",0.169804,0.645152,0.595294,0.014242,32
+1,Mayor/County Executive/Governor of each of the existing partnership,0.169804,0.661212,0.555294,0.013636,33
+1,communities. Although the Mayor/County Executive/Governor may request input,0.16902,0.677879,0.647451,0.013636,34
+1,"from, or may be given input by, the sister cities program, it is up to the discretion",0.168627,0.693939,0.647059,0.013939,35
+1,of the Mayor/County Executive/Governor to sign the cooperative agreement.,0.16902,0.709697,0.612941,0.013939,36
+1,Although Sister Cities International will help with the cooperative agreement,0.168627,0.726364,0.605882,0.013636,37
+1,"process, it is up to the requesting community to get the agreement signed. Sister",0.169412,0.742121,0.650196,0.013939,38
+1,"Cities International will not, in any way, force a community to ""share"" and sign",0.16902,0.758182,0.623922,0.014242,39
+1,the cooperative agreement.,0.168627,0.774848,0.219216,0.013333,40
+1,"To place a relationship into Emeritus status, the mayor or highest elected official of the U.S.",0.117255,0.798485,0.736471,0.013939,41
+1,community must write a letter to the mayor of the foreign city indicating that they wish to,0.118039,0.814545,0.70902,0.013636,42
+1,"remain sister cities, but understand that the relationship will remain inactive until such time as",0.118039,0.831212,0.747451,0.013333,43
+1,both cities are able to sustain an active relationship. Sister Cities International should be,0.118039,0.847273,0.705098,0.013636,44
+1,informed in writing by the mayor of the U.S. city of the situation. Sister Cities International will,0.118039,0.863333,0.746275,0.013636,45
+2,Partnership Agreement,0.516078,0.027879,0.440784,0.032424,1
+2,SisterCities,0.169804,0.033333,0.238824,0.028182,2
+2,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788,3
+2,Toolkit,0.83098,0.072727,0.127059,0.025455,4
+2,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303,5
+2,then place the partnership into Emeritus Status and will reflect this status in directories and all,0.117255,0.132424,0.751373,0.013333,6
+2,lists of sister city programs.,0.118039,0.148788,0.218431,0.013333,7
+2,"If a community wishes to terminate a sister city relationship, then a letter from the mayor or",0.118431,0.172424,0.732549,0.013333,8
+2,highest elected official of the U.S. city should be sent to the mayor of the sister city. Sister,0.118039,0.188485,0.721569,0.013636,9
+2,Cities International should be informed of this action in writing by the mayor of the U.S. city,0.118039,0.204848,0.72902,0.013333,10
+2,and Sister Cities International will then remove the partnership from its directories and all lists,0.117647,0.221212,0.746275,0.013333,11
+2,of sister city programs. We do not recommend terminating a relationship simply because it is,0.117647,0.237273,0.743529,0.013333,12
+2,"dormant. Many partnerships wax and wane over the years, and in many cases a dormant",0.117647,0.253939,0.713333,0.013333,13
+2,partnership may be reinvigorated by local members years after it has been inactive.,0.118039,0.269697,0.664314,0.013636,14
+2,General Guidelines,0.118039,0.295152,0.231765,0.016061,15
+2,In order for a sister city/county/state partnership to be recognized by Sister Cities International,0.118431,0.324242,0.754902,0.013636,16
+2,"(SCI), the two communities must sign formal documents which clearly endorse the link. This",0.118039,0.340606,0.74,0.013636,17
+2,presumes several key items: that the U.S. community is already a member of SCI and has,0.118039,0.35697,0.718039,0.013636,18
+2,followed proper procedures (e.g. passed a city council resolution declaring the intent to twin,0.117255,0.373333,0.737647,0.013636,19
+2,with the specific city); that both communities share a mutual commitment to the relationship;,0.117255,0.389394,0.740784,0.013636,20
+2,and that both have secured the necessary support structure to build a lasting relationship. You,0.117647,0.405455,0.758039,0.013333,21
+2,should check with your local sister city program to see if they have any additional requirements,0.117647,0.421818,0.760784,0.013636,22
+2,before pursuing a sister city relationship.,0.118039,0.437879,0.323137,0.013636,23
+2,"SCI often refers to these agreements as a ""Sister City Agreement"" or ""Memorandum of",0.118039,0.461515,0.696863,0.013939,24
+2,"Understanding."" However, as the following examples show, the actual name and format of",0.118039,0.477576,0.729804,0.013636,25
+2,your documents is left up to you.,0.117255,0.494242,0.262745,0.013636,26
+2,A few things to keep in mind as you draft your agreement:,0.117255,0.517879,0.463137,0.013636,27
+2,"Your agreement can range from the ceremonial, with language focusing on each city's",0.176471,0.542121,0.69098,0.013939,28
+2,"commitment to fostering understanding, cooperation, and mutual benefit to the precise,",0.176471,0.558485,0.701961,0.013333,29
+2,"with particular areas of interest, specific programs/activities, or more concrete goals",0.176078,0.574848,0.673725,0.013636,30
+2,related to anything from numbers of exchanges to economic development.,0.176863,0.591212,0.596863,0.013636,31
+2,"Don't try to include everything you plan to do. Some specifics, like particular areas of",0.177255,0.620303,0.681176,0.013939,32
+2,"interest or participating institutions are good to include. However, there's no need to",0.176471,0.636667,0.675686,0.013636,33
+2,include all the programs you plan to do if it makes the document too lengthy or limits,0.176863,0.652727,0.678824,0.013939,34
+2,the scope of projects. This is a formal document to establish the relationship; specific,0.176078,0.668788,0.684706,0.013636,35
+2,"tasks, responsibilities, or other nuts-and-bolts text related to implementation or",0.176078,0.685455,0.635686,0.013333,36
+2,administration of the partnership can be expressed more fully in a separate,0.176471,0.701212,0.600392,0.013636,37
+2,memorandum between the respective sister city committees. Your partnership,0.177255,0.717576,0.626667,0.013636,38
+2,agreement is a historical document and should not be dated or limited by being aligned,0.176471,0.733636,0.699216,0.013636,39
+2,with very specific tasks.,0.176078,0.750606,0.190196,0.013333,40
+2,Work with your counterparts. Remember that this is signed by both cities. You should,0.176078,0.779697,0.68549,0.013636,41
+2,share drafts of your agreement with your international partners and solicit feedback on,0.176471,0.795758,0.691765,0.013333,42
+2,what they'd like to see in the agreement. Be flexible to cultural or municipal priorities.,0.176471,0.811818,0.679216,0.013939,43
+2,Ask your counterparts to translate the agreement if it is drafted in English. It is,0.176078,0.841515,0.623137,0.013636,44
+2,important for the citizens of your partner community to be able to read and understand,0.176863,0.857576,0.693725,0.013939,1
+2,the commitment their city has made. Have someone in your own community who,0.176078,0.873939,0.649804,0.013636,2
+3,Partnership Agreement,0.516078,0.027879,0.441176,0.032121,3
+3,SisterCities,0.169804,0.033333,0.239216,0.028182,4
+3,INTERNATIONAL,0.170196,0.06697,0.237255,0.008788,5
+3,Toolkit,0.83098,0.07303,0.126667,0.025152,6
+3,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303,7
+3,speaks that language check the foreign-language version to make sure it mirrors what,0.176471,0.132424,0.688235,0.013333,8
+3,you have in your own agreement.,0.176471,0.148788,0.264706,0.013333,9
+3,Keep it to one page. Ceremonial documents such as these partnership agreements,0.176863,0.178485,0.66549,0.013636,10
+3,work best if they can be posted in their entirety.,0.176078,0.194545,0.380392,0.013636,11
+3,Most sister city agreements include some acknowledgement of the founding principles,0.177255,0.224242,0.694902,0.013636,12
+3,"of the sister city movement- to promote peace through mutual respect, understanding,",0.176471,0.240303,0.698431,0.013333,13
+3,and cooperation.,0.176471,0.25697,0.13451,0.013333,14
+3,Consider using official letterhead and/or other embellishments such as city seals or,0.176863,0.286061,0.665882,0.013333,15
+3,logos to reflect your enhance the document. Sister city agreements are often posted at,0.176863,0.302121,0.695686,0.013636,16
+3,city hall or other municipal offices and should reflect their historical importance,0.176471,0.318485,0.630588,0.013333,17
+3,Look at other agreements your city has signed. These agreements may give you an idea,0.177255,0.347879,0.705098,0.013636,18
+3,"of what is acceptable or possible, and they may be in an easily replicable format. If you",0.176471,0.364242,0.695686,0.013636,19
+3,"cannot access older agreements please contact Sister Cities International, we may",0.176863,0.380303,0.663137,0.013636,20
+3,"have them on file, although we do not have copies of all partnership agreements.",0.176863,0.396667,0.64549,0.013636,21
+3,Documents must be signed by the top elected official of both communities.,0.177255,0.426364,0.601569,0.013333,22
+3,"Check with your mayor, city council, town clerk, et al. to make sure that the agreement",0.176863,0.455758,0.694118,0.013636,23
+3,"is OK with them. The mayor is the one putting his or her name on the paper, and you",0.176863,0.471818,0.677255,0.013333,24
+3,don't want to spend time developing an agreement which will never be signed.,0.176863,0.488182,0.629412,0.013636,25
+3,Official documents are usually signed during a formal ceremony recognizing the,0.176863,0.517576,0.638431,0.013636,26
+3,partnership. Be sure both communities receive a signed set of the official documents,0.177255,0.533939,0.683922,0.013636,27
+3,for their records.,0.176078,0.550606,0.131373,0.010606,28
+3,Remember to send your signed agreement to Sister Cities International. After we,0.177255,0.579697,0.645098,0.013636,29
+3,receive your agreement we will post the relationship in the City Directory and make sure,0.176863,0.595758,0.703137,0.013636,30
+3,it is included in our Annual Membership Directory.,0.176863,0.612121,0.398039,0.013333,31
+3,Remember that each city's sister city program is independent and can impose requirements,0.118431,0.640606,0.736471,0.013939,32
+3,"like the establishment of a committee, a review period, sustainability/funding plan, among",0.118039,0.65697,0.715686,0.013636,33
+3,"others, before sanctioning a sister city agreement. Check with your local program or mayor's",0.117647,0.672727,0.743529,0.014242,34
+3,office to see if this is the case.,0.117647,0.689091,0.241176,0.011515,35
+3,On the following pages you'll find a series of partnership agreements to give you an idea of,0.118039,0.717879,0.728627,0.013939,36
+3,"what is possible. While you should feel free to use some of the formatting and language, we",0.117255,0.734242,0.73451,0.013636,37
+3,encourage you to make your agreement your own and be creative with what you produce. If,0.117647,0.750606,0.737647,0.013636,38
+3,you are unsure about your agreement or want advice you can always solicit feedback by,0.117647,0.766667,0.708627,0.013636,39
+3,sending it to our Membership Director at akaplan@sister-cities.org or contacting us at (202),0.117647,0.782727,0.732157,0.013636,40
+3,347-8630.,0.117647,0.799394,0.080392,0.010303,41
+4,Partnership Agreement,0.516471,0.027879,0.440784,0.032727,1
+4,SisterCities,0.169412,0.033333,0.239608,0.028485,2
+4,INTERNATIONAL,0.170196,0.066667,0.238431,0.009091,3
+4,Toolkit,0.830588,0.072727,0.127843,0.025758,4
+4,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333,5
+4,"jull bubzig 2000 3,312",0.378039,0.291212,0.32549,0.019394,6
+4,ABU DHABI MUNICIPALITY & TOWN PLANNING,0.376471,0.316667,0.327451,0.016667,7
+4,AN AGREEMENT FOR THE ESTABLISHMENT OF,0.260784,0.373636,0.52549,0.012727,8
+4,SISTER CITIES RELATIONSHIP,0.337647,0.393636,0.342745,0.012121,9
+4,BETWEEN,0.454902,0.413636,0.110588,0.011212,10
+4,THE CITY OF ABU DHABI ( U. A.E),0.337255,0.432727,0.375686,0.013939,11
+4,AND,0.487843,0.452727,0.048235,0.011212,12
+4,"HOUSTON, TEXAS ( U.S.A)",0.385882,0.471515,0.298039,0.014848,13
+4,"The Sister City Program, administered by Sister Cities International, was initiated",0.221961,0.525455,0.597255,0.01303,14
+4,By the President of the United States of America in 1956 to encourage greater,0.222745,0.539394,0.561961,0.012727,15
+4,Friendship and understanding between the United States and other nations through,0.222745,0.553333,0.608235,0.012727,16
+4,Direct personal contact: and,0.222745,0.567576,0.20549,0.012424,17
+4,"In order to foster those goals, the people of Abu Dhabi and Houston, in a gesture of",0.222353,0.594242,0.603529,0.012424,18
+4,"Friendship and goodwill, agree to collaborate for the mutual benefit of their",0.222745,0.608182,0.547843,0.01303,19
+4,"Communities by exploring education, economic and cultural opportunities.",0.222353,0.622121,0.541961,0.012121,20
+4,"Abu Dhabi and Houston, sharing a common interest in energy, technology and",0.221569,0.648788,0.574118,0.012424,21
+4,"medicine, and the desire to promote mutual understanding among our citizens do",0.222353,0.66303,0.588235,0.012121,22
+4,"hereby proclaim themselves Sister Cities beginning on the 13th day of March 2001,",0.221961,0.673636,0.594118,0.015758,23
+4,the date of Houston City Council resolution estatblishing the Sister City,0.221961,0.690303,0.519608,0.01303,24
+4,relationship became effective.,0.221569,0.705152,0.217647,0.012424,25
+4,"Signed on this 26 of October 2002, in duplicate in the Arabic and English",0.221569,0.732121,0.533333,0.01303,26
+4,"Languages, both text being equally authentic.",0.221961,0.746667,0.328627,0.012727,27
+4,A,0.344314,0.768485,0.084706,0.030303,28
+4,Sheikh Mohammed bin Butti AI Hamed,0.245882,0.806364,0.366275,0.010909,29
+4,Lee P.Brown,0.729412,0.806364,0.118824,0.010303,30
+4,Mayor of Houston,0.704706,0.823333,0.166667,0.012424,31
+4,Chairman of Abu Dhabi Municipality,0.24549,0.823636,0.342353,0.012727,32
+4,&Town Planning,0.324314,0.841212,0.155686,0.012424,33
+5,Partnership Agreement,0.516078,0.027879,0.441176,0.032424,1
+5,SisterCities,0.169412,0.033333,0.239608,0.028485,2
+5,INTERNATIONAL,0.17098,0.066667,0.237255,0.009091,3
+5,Toolkit,0.83098,0.072727,0.127059,0.025758,4
+5,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333,5
+5,THE CITY OF NEW YORK,0.438824,0.262121,0.240784,0.009697,6
+5,OFFICE OF THE MAYOR,0.450196,0.27697,0.220392,0.009697,7
+5,"NEW YORK, N.Y. 10007",0.461176,0.29303,0.196863,0.010303,8
+5,THE NEW YORK CITY-LONDON SISTER CITY PARTNERSHIP,0.267451,0.355758,0.582745,0.011818,9
+5,Memorandum of Understanding,0.420392,0.371212,0.274902,0.013333,10
+5,The Sister City partnership between New York City and London will foster mutually,0.201176,0.402121,0.674118,0.014242,11
+5,beneficial solutions to common challenges for these two great cosmopolitan entities.,0.201176,0.417273,0.66902,0.013636,12
+5,"Consequently, the Sister City relationship between the two will be one of the most",0.201176,0.432727,0.652549,0.015152,13
+5,"important in their network of global partnerships, as it strives to:",0.201176,0.448182,0.50902,0.015455,14
+5,Encourage and publicize existing exchanges between London and New York City so,0.230588,0.480303,0.671373,0.015152,15
+5,that they can flourish to benefit a wider cross-section of the citizens of both;,0.230588,0.496061,0.602353,0.015152,16
+5,"Support and promote the development of new social, economic, academic and",0.230196,0.512424,0.618431,0.015455,17
+5,community programs to encourage both cities' citizens to share their experiences as a,0.229804,0.527879,0.678039,0.014848,18
+5,medium for learning from one another;,0.229804,0.543636,0.309412,0.013939,19
+5,Generate an improvement of the operation of the cities' various government agencies,0.229804,0.56,0.676078,0.014545,20
+5,by serving as a conduit of information;,0.22902,0.575758,0.307843,0.014848,21
+5,"Identify themes, common to both, that can generate new initiatives to further and",0.229412,0.591818,0.640784,0.015152,22
+5,"nurture the increasingly powerful financial, social and cultural relationships between",0.22902,0.607576,0.671373,0.014242,23
+5,the cities;,0.22902,0.624545,0.076471,0.012424,24
+5,Promote key mayoral priorities relevant to both London and New York City;,0.228627,0.639394,0.608627,0.015152,25
+5,Provide financial or in kind support to community-led programs that advance the,0.228627,0.656061,0.641569,0.013636,26
+5,aims of the Sister City partnership;,0.22902,0.672121,0.275294,0.013636,27
+5,"With the above purposes in mind, the Mayor of the City of New York and the Mayor of",0.198824,0.702424,0.697647,0.014848,28
+5,London solemnly confirm that these two cities are united by an official partnership by the,0.198824,0.718182,0.710196,0.014545,29
+5,protocol of this Memorandum of Understanding.,0.198431,0.733939,0.384314,0.015152,30
+5,This agreement will go into effect from the date of signatures.,0.310196,0.780606,0.488235,0.014545,31
+5,Thedder Rudolph W. Giuliani,0.178824,0.795455,0.244314,0.100909,32
+5,Signed in March of 2001,0.455686,0.796364,0.19451,0.013636,33
+5,Ken Mayor Livingstone,0.672157,0.877576,0.132941,0.029091,34
+5,Mayor,0.311373,0.894848,0.053333,0.012727,35
+5,New York City,0.287843,0.909091,0.121176,0.013333,36
+5,London,0.701961,0.909091,0.061569,0.010606,37
+6,Partnership Agreement,0.515686,0.027576,0.441961,0.03303,1
+6,SisterCities,0.169412,0.03303,0.24,0.028182,2
+6,INTERNATIONAL,0.169804,0.066667,0.238431,0.009091,3
+6,Toolkit,0.83098,0.072727,0.127451,0.025758,4
+6,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333,5
+6,CHIC OF STATE,0.247451,0.190606,0.141961,0.036364,6
+6,City of Long Beach,0.388627,0.196667,0.476471,0.066364,7
+6,California,0.551373,0.257273,0.136471,0.033333,8
+6,Sister City Agreement,0.321961,0.305455,0.378431,0.035152,9
+6,between the,0.464706,0.352727,0.084314,0.009697,10
+6,City of Long Beach,0.38,0.378485,0.252549,0.01697,11
+6,"California, USA",0.4,0.397576,0.21098,0.016061,12
+6,and the,0.48,0.415152,0.053333,0.009091,13
+6,City of San Pablo de Manta,0.321569,0.428788,0.369804,0.01697,14
+6,"Ecuador, South America",0.347451,0.447879,0.317255,0.015152,15
+6,"In accordance with the authorization and approval expressed by the City of Long Beach,",0.261569,0.482121,0.536863,0.012121,16
+6,"California, USA, and the City of San Pablo de Manta, Ecundor, South America, it is declared",0.217647,0.492727,0.581176,0.01303,17
+6,"that a ""Sister City Agreement between the two cities is hereby established for the following",0.217647,0.502727,0.581569,0.012121,18
+6,purposes:,0.216863,0.516061,0.058039,0.009394,19
+6,(1) to promote and expand the effective and mutually beneficial cooperation between,0.278824,0.532727,0.520392,0.012424,20
+6,the people of Long Beach and the people of San Pablo de Manta; and,0.218039,0.543636,0.40549,0.012424,21
+6,"(2) to promote international goodwill, understanding, and expanded business",0.279216,0.56303,0.520784,0.012424,22
+6,"relations between the two cities and their respective nations by the exchange of people, ideas, and",0.218039,0.573636,0.581569,0.012121,23
+6,"information in a unide variety of economic, social, cultural, municipal, environmental,",0.218039,0.584242,0.581176,0.012121,24
+6,"professional, technical, youth, and other endeavors; and",0.217647,0.594848,0.333333,0.012121,25
+6,"(3) to foster and encourage charitable, scientific, trade and commerce, literary and",0.279608,0.613939,0.520784,0.012727,26
+6,educational activities between the two cities;,0.218039,0.625455,0.265882,0.009697,27
+6,This Sister City Agreement shall be officially established and shall become effective when,0.263137,0.644545,0.536863,0.012727,28
+6,"this document has been duly executed by the Mayor of Long Beach, California, USA, and the",0.218824,0.654848,0.581961,0.012424,29
+6,"Mayor of San Pablo de Manta, Ecundor, South America.",0.218431,0.665758,0.338824,0.012121,30
+6,STATE OFFICE,0.276471,0.713636,0.050588,0.048788,31
+6,Beverly 0 Neill,0.587451,0.736667,0.121961,0.013636,32
+6,"Mayor, City of Long Beach",0.542353,0.751212,0.21098,0.013636,33
+6,"California, USA",0.582745,0.765758,0.125098,0.01303,34
+6,10.2aulus,0.490588,0.771818,0.220392,0.062424,35
+6,Ing. Jorge O. Zambrano Cedeño,0.527059,0.825152,0.242745,0.013333,36
+6,"Mayor, City of San Pablo de Manta",0.505098,0.839394,0.277647,0.013636,37
+6,"Ecuador, South America",0.551765,0.854242,0.188235,0.011818,38
+6,"Dated: September 19, 2000",0.544706,0.883333,0.202745,0.01303,39
+7,Partnership Agreement,0.516078,0.027879,0.441176,0.032424,1
+7,SisterCities,0.169412,0.03303,0.24,0.028485,2
+7,INTERNATIONAL,0.170196,0.066667,0.237647,0.009091,3
+7,Toolkit,0.83098,0.072727,0.127451,0.025758,4
+7,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333,5
+7,REAFFIRMATION OF SISTER CITIES DECLARATION,0.324706,0.165152,0.483529,0.013939,6
+7,adopted by,0.2,0.213333,0.080392,0.013636,7
+7,THE HONORABLE RICHARD M. DALEY,0.396078,0.214242,0.335686,0.012424,8
+7,MAYOR OF CHICAGO,0.472549,0.231212,0.18549,0.011515,9
+7,and,0.199608,0.260909,0.026275,0.010606,10
+7,THE HONORABLE ZHANG RONGMAO,0.401961,0.261212,0.323137,0.011212,11
+7,MAYOR OF SHENYANG,0.463529,0.273636,0.202353,0.011212,12
+7,ON,0.551765,0.298182,0.026667,0.011515,13
+7,"JUNE 5, 1995",0.500392,0.323636,0.128235,0.014848,14
+7,"On this the tenth anniversary of the signing of a sister city agreement, in order to further",0.255686,0.36303,0.67098,0.015152,15
+7,the traditional links of friendship between Chicago and Shenyang and to reaffirm their mutual,0.198824,0.378788,0.727843,0.015455,16
+7,"aspiration to work in unison for the benefit of their cities and nations, the Honorable Mayor",0.199608,0.394848,0.727843,0.014848,17
+7,"Richard M. Daley, Mayor of the City of Chicago, and the Honorable Zhang Rongmao, Mayor",0.199216,0.411212,0.727451,0.014242,18
+7,"of the City of Shenyang, on this fifth day of June 1995, do hereby acknowledge and reaffirm the",0.199216,0.42697,0.72549,0.014848,19
+7,sister cities agreement between the City of Chicago and the City of Shenyang.,0.199608,0.443636,0.57451,0.014242,20
+7,"The City of Chicago and the City of Shenyang on the basis of friendly cooperation,",0.256078,0.473939,0.665098,0.015152,21
+7,equality and mutual benefit will continue to develop a sister cities relationship to promote and,0.2,0.490303,0.724706,0.014242,22
+7,broaden economic cooperation and cultural exchanges between the two cities.,0.199216,0.506061,0.57451,0.014242,23
+7,The two cities do hereby declare their interest in exploring the establishment of business,0.255294,0.537273,0.668235,0.015455,24
+7,and trade relations between Chicago and Shenyang.,0.198824,0.554545,0.387843,0.013636,25
+7,"In addition, exchanges will be promoted in the area of the arts such as exhibits, music,",0.254118,0.583939,0.666667,0.015455,26
+7,dance and other cultural activities.,0.198431,0.601212,0.256471,0.010606,27
+7,"In addition, exchanges will be promoted in education and the establishment of contacts",0.254118,0.630303,0.668627,0.015758,28
+7,within educational institutions encouraged.,0.198824,0.647273,0.32,0.014242,29
+7,"In addition, we declare our intention to promote exchanges in such fields as science and",0.253725,0.678182,0.668627,0.014848,30
+7,"technology, sports, health, youth and any areas that will contribute to the prosperity and the",0.198039,0.693636,0.722745,0.015152,31
+7,further development of friendship between the people of our two cities.,0.194902,0.711515,0.525098,0.013636,32
+7,3h.5.,0.593725,0.750606,0.218039,0.06303,33
+7,THE HONORABLE ZHANG RONGMAO,0.588627,0.819394,0.287843,0.011818,34
+7,THE HONORABLE RICHARD M. DALEY,0.197255,0.821515,0.303529,0.010606,35
+7,MAYOR OF SHENYANG,0.587451,0.835455,0.177647,0.010303,36
+7,MAYOR OF CHICAGO,0.195686,0.835758,0.164706,0.010606,37

doc_redaction/example_data/example_outputs/Partnership-Agreement-Toolkit_0_0.pdf_review_file.csv

@@ -0,0 +1,77 @@
+image,page,label,color,xmin,ymin,xmax,ymax,id,text
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_0.png,1,ADDRESS,"(0, 0, 0)",0.598431,0.524545,0.63098,0.535455,EG3nykuwvxbk,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_0.png,1,ADDRESS,"(0, 0, 0)",0.820392,0.798485,0.854118,0.809394,jy1R42e6phNz,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_0.png,1,ADDRESS,"(0, 0, 0)",0.433333,0.863333,0.46549,0.873939,9sbrsroLfZy0,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_1.png,2,ADDRESS,"(0, 0, 0)",0.354118,0.188788,0.386275,0.199697,k7bWBsQQchJZ,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_1.png,2,ADDRESS,"(0, 0, 0)",0.780392,0.204848,0.812941,0.215758,peo6UqIxrjmR,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_2.png,3,EMAIL,"(0, 0, 0)",0.447843,0.78303,0.648627,0.796667,DIfz0LenOtQv,akaplan@sister-cities.org
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_2.png,3,PHONE,"(0, 0, 0)",0.809804,0.78303,0.850196,0.796667,odJdySe9XrAn,(202)
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_2.png,3,PHONE,"(0, 0, 0)",0.117647,0.799394,0.198431,0.809697,iURSkUM7BbUG,347-8630
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.637647,0.432727,0.712941,0.44697,fRxAD9qm856s,U. A.E
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.489412,0.43303,0.614902,0.444545,qzRFPlNbslpH,ABU DHABI
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.385882,0.472121,0.593725,0.486364,v1uLbGsofN1f,"HOUSTON, TEXAS"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.392549,0.539697,0.573725,0.549394,MvbPQiHvSdL7,United States of America
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.539216,0.553333,0.635686,0.563333,05U3cgj5w9PY,United States
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.534902,0.594242,0.615294,0.603939,uHMikyBlMq5f,Abu Dhabi
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.651373,0.594242,0.717255,0.605455,XNUE0GopIBaf,Houston
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.221569,0.65,0.301176,0.659697,6FjbNu2CGA9n,Abu Dhabi
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.337647,0.65,0.404314,0.660606,Yvmm2225ityu,Houston
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,HANDWRITING,"(0, 0, 0)",0.344314,0.768485,0.42902,0.798788,EwTcqq7PENU8,A
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.245882,0.806364,0.612549,0.817576,Mj4gqwbgsZWp,Sheikh Mohammed bin Butti AI Hamed
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.52,0.806364,0.612549,0.81697,RXYOVgLwq8Ke,AI Hamed
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.729412,0.806364,0.848235,0.816667,REPZhwFWGoTc,Lee P.Brown
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.245882,0.806667,0.51451,0.817576,rFdxMRFRWLRJ,Sheikh Mohammed bin Butti
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.366667,0.823939,0.465098,0.834242,5iYCxRGdPG1i,Abu Dhabi
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.577647,0.262121,0.68,0.271515,3ZR43H3yYNdy,NEW YORK
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.461176,0.29303,0.555294,0.303333,WNoitmR9A6lu,NEW YORK
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.461176,0.29303,0.658039,0.303333,HjrhxMQhovlF,NEW YORK N.Y. 10007
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.563137,0.29303,0.658039,0.302121,nPN7g7UcnX4u,N.Y. 10007
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.314118,0.356667,0.42549,0.367576,ZoJf29CB3Wrq,NEW YORK
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.655294,0.480909,0.718431,0.491515,iezAqmD2ilnb,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.708627,0.639394,0.837255,0.652727,tWAuJEQVpfhi,New York City
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.60902,0.64,0.67098,0.650606,NaW3mmmlhMW9,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.667059,0.702727,0.751373,0.713636,pgMiwuMiBp8B,New York
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.198824,0.720303,0.261569,0.731212,fPvElSFZFRoL,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,HANDWRITING,"(0, 0, 0)",0.178824,0.795455,0.281961,0.896364,DfniF7P2bXAw,Thedder
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,NAME,"(0, 0, 0)",0.178824,0.795455,0.423529,0.896364,QwnWsAeslO5f,Thedder Rudolph W. Giuliani
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,NAME - ADDRESS,"(0, 0, 0)",0.672157,0.877576,0.80549,0.891212,Vdp95SShYOEO,Ken Livingstone
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.710196,0.877576,0.80549,0.891212,H5DGqsucPAjc,Livingstone
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,NAME,"(0, 0, 0)",0.672157,0.877879,0.705098,0.888182,qotGtnMbhAJr,Ken
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.287843,0.909091,0.40902,0.922727,sFX0tNJJzpE5,New York City
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.701961,0.909091,0.763922,0.919697,2xFbVTbxiOhC,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.55451,0.203636,0.86549,0.258485,Nfe3WTBembGQ,Long Beach
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.551373,0.257273,0.687843,0.290606,kndQY5X4itc8,California
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.558824,0.397879,0.611373,0.410303,B5vq8yhWLeOg,USA
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.425882,0.429091,0.691373,0.441818,OtNgqUkoEaZb,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.347451,0.447879,0.665098,0.46303,Q52VzBx2SWNF,"Ecuador, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.724314,0.482121,0.798431,0.493939,O7gd9ywvKsKh,"Long Beach,"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.425098,0.49303,0.506275,0.502727,DzYr3xrM8Tvv,San Pablo de
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.425098,0.49303,0.715294,0.50303,iZ0knpQD54UU,"San Pablo de Manta, Ecundor, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.509804,0.49303,0.715294,0.50303,pZnYGzr7Pwsl,"Manta, Ecundor, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.217647,0.493333,0.321961,0.504242,r7Aar8FNQF6D,"California, USA"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.471765,0.543636,0.596863,0.553939,zg9uBDlSuuA1,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.295294,0.544242,0.36549,0.556061,A0OY6RjMEocW,Long Beach
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.563137,0.655152,0.748627,0.667576,HQlTdEUhOCgI,"Long Beach, California, USA"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.463529,0.665758,0.557255,0.674848,bCN9b7kJw0Ik,South America
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.277647,0.666061,0.403529,0.676061,qffN3bDgWRMk,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.587451,0.736667,0.709804,0.750303,eqMENFw5mbnL,Beverly 0 Neill
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.663137,0.751212,0.753333,0.764545,POqPQVBCES8h,Long Beach
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.582745,0.765758,0.708235,0.779091,mjrjsSMOxwaY,"California, USA"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,HANDWRITING,"(0, 0, 0)",0.490588,0.771818,0.71098,0.834242,xL8dSawihWuY,10.2aulus
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,NAME,"(0, 0, 0)",0.559608,0.825152,0.769804,0.838485,fHyvwmbOgLMJ,Jorge O. Zambrano Cedeño
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.624314,0.839394,0.782745,0.850303,zGhskyehufSv,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.551765,0.854242,0.74,0.866061,dSPXmtb8M4nt,"Ecuador, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.556471,0.215152,0.731765,0.226667,BEhuvaI5BVaR,RICHARD M. DALEY
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.563137,0.261212,0.725098,0.272424,coo8KK7q6A72,ZHANG RONGMAO
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.566275,0.273636,0.666275,0.285152,0P9rVSbeNdB4,SHENYANG
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.526667,0.380303,0.588235,0.394242,1GDArufutI5y,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.628235,0.380606,0.702353,0.394242,QyD751r4fCU1,Shenyang
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.736863,0.411515,0.868235,0.424545,rntIekANI8BO,Zhang Rongmao
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.199216,0.411818,0.34,0.424848,96TaHazXGIM7,Richard M. Daley
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.514902,0.412424,0.580784,0.425758,kbyVj6qhZSPi,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.696471,0.443939,0.774118,0.45697,rJpaMvepsNln,Shenyang
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.353725,0.474545,0.415686,0.489091,PokCVpLQmDki,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.407451,0.554545,0.469804,0.568182,HqVr414KRg59,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,HANDWRITING,"(0, 0, 0)",0.593725,0.750606,0.811765,0.813636,xdawEv0DUH6P,3h.5.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.730196,0.819394,0.876471,0.830606,Gghr7ccN6lS2,ZHANG RONGMAO
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.34,0.821515,0.501176,0.831515,vOMIv1RS5Sag,RICHARD M. DALEY

doc_redaction/example_data/example_outputs/Partnership-Agreement-Toolkit_0_0_ocr_output_local_ocr.csv

@@ -0,0 +1,1241 @@
+page,text,left,top,width,height,line,conf
+1,SisterCities Partnership Agreement,0.167843,0.025758,0.793725,0.03697,1,99.0
+1,IN,0.17098,0.066667,0.040392,0.009091,2,92.0
+1,Connect,0.167843,0.085152,0.068627,0.015455,3,98.0
+1,Types of,0.115686,0.153939,0.114118,0.024242,4,98.0
+1,Sister City,0.116471,0.185758,0.095686,0.015455,5,97.0
+1,A Sister City,0.116078,0.209394,0.101569,0.015152,6,98.0
+1,"do not take place,",0.116863,0.22697,0.147451,0.013333,7,99.0
+1,another country,0.116863,0.242424,0.132549,0.013939,8,98.0
+1,"""sister city/sister",0.116863,0.258485,0.156078,0.013636,9,99.0
+1,unless otherwise,0.116863,0.274848,0.141961,0.013939,10,99.0
+1,Sister Cities,0.116863,0.298485,0.099608,0.014545,11,99.0
+1,good standing,0.116471,0.314545,0.116078,0.014545,12,99.0
+1,"website. However,",0.115686,0.330606,0.149412,0.014848,13,99.0
+1,legitimacy of,0.116863,0.347576,0.106667,0.013636,14,98.0
+1,Friendship City,0.116078,0.370303,0.129412,0.016061,15,98.0
+1,A Friendship City,0.115686,0.393939,0.139608,0.015758,16,97.0
+1,"stone"" to a more",0.116078,0.410909,0.138039,0.014545,17,99.0
+1,referred to,0.116863,0.427273,0.108235,0.013333,18,99.0
+1,recognize,0.116863,0.443333,0.086275,0.014242,19,98.0
+1,As per Sister,0.117255,0.467576,0.106667,0.013333,20,99.0
+1,Sister,0.167451,0.490606,0.052941,0.014242,21,97.0
+1,U.S. and,0.167843,0.50697,0.070588,0.014242,22,98.0
+1,between,0.167451,0.523333,0.071765,0.013939,23,98.0
+1,cooperative,0.167451,0.539091,0.102745,0.015455,24,99.0
+1,International.,0.168235,0.555758,0.120392,0.013939,25,98.0
+1,is received,0.167451,0.571515,0.094118,0.013636,26,99.0
+1,recognize,0.168235,0.587879,0.085098,0.014545,27,99.0
+1,with a,0.168627,0.604848,0.05098,0.013333,28,99.0
+1,The,0.168235,0.627879,0.034902,0.014545,29,99.0
+1,Mayor/County,0.169804,0.660909,0.116863,0.013939,30,97.0
+1,communities.,0.167843,0.67697,0.107843,0.013939,31,99.0
+1,"from, or",0.168627,0.693333,0.072941,0.013333,32,98.0
+1,of the,0.168627,0.709697,0.053333,0.012424,33,99.0
+1,Although,0.168235,0.725152,0.073333,0.013939,34,99.0
+1,"process,",0.168235,0.741818,0.084706,0.013939,35,98.0
+1,Cities,0.167843,0.75697,0.048235,0.015152,36,98.0
+1,the,0.168627,0.774545,0.029804,0.013333,37,99.0
+1,To place a,0.116078,0.796667,0.087059,0.015152,38,98.0
+1,community must,0.116863,0.813939,0.134902,0.014242,39,98.0
+1,remain sister,0.116863,0.83,0.113333,0.013333,40,98.0
+1,both cities are,0.116471,0.845758,0.12,0.013636,41,98.0
+1,informed in,0.116863,0.861818,0.096863,0.014848,42,98.0
+1,TERN ATION,0.211373,0.066667,0.164314,0.009091,43,92.0
+1,globally. Thrive,0.236471,0.085152,0.121961,0.015455,44,98.0
+1,Affiliations,0.229804,0.153939,0.131373,0.024242,45,98.0
+1,Relationship,0.212157,0.185758,0.103137,0.015455,46,97.0
+1,relationship is,0.217647,0.209394,0.115294,0.015152,47,98.0
+1,highest,0.264314,0.22697,0.064314,0.013333,48,99.0
+1,or territory,0.249412,0.242424,0.121176,0.013939,49,98.0
+1,"cities""",0.272941,0.258485,0.048627,0.013636,50,99.0
+1,indicated,0.258824,0.274848,0.076863,0.013939,51,99.0
+1,International shall,0.216471,0.298485,0.150196,0.014545,52,99.0
+1,(i.e. who are,0.232549,0.314545,0.103137,0.014545,53,99.0
+1,Sister Cities,0.265098,0.330606,0.101569,0.014848,54,99.0
+1,those relationships,0.223529,0.347576,0.157647,0.013636,55,98.0
+1,or Friendship,0.255294,0.393939,0.113725,0.015758,56,97.0
+1,"formal ""Sister",0.254118,0.410909,0.121961,0.014545,57,99.0
+1,as such in the,0.225098,0.427273,0.103922,0.013333,58,99.0
+1,Friendship City,0.203137,0.443333,0.125882,0.014242,59,98.0
+1,Cities International,0.223922,0.467576,0.151373,0.013333,60,99.0
+1,Cities International,0.220392,0.490606,0.156863,0.014242,61,97.0
+1,an international,0.238431,0.50697,0.128235,0.014242,62,98.0
+1,that international,0.239216,0.523333,0.144314,0.013939,63,98.0
+1,agreement,0.270196,0.539091,0.092549,0.015455,64,99.0
+1,If a,0.288627,0.555758,0.029804,0.013939,65,98.0
+1,within a,0.261569,0.571515,0.063529,0.013636,66,99.0
+1,the partnership,0.253333,0.587879,0.130196,0.014545,67,99.0
+1,symbol in the,0.219608,0.604848,0.110196,0.013333,68,99.0
+1,cooperative,0.203137,0.627879,0.102745,0.014545,69,99.0
+1,Executive/Governor,0.169804,0.644848,0.179216,0.013333,70,99.0
+1,Although,0.275686,0.67697,0.077647,0.013939,71,99.0
+1,may be given,0.241569,0.693333,0.108235,0.013333,72,98.0
+1,Mayor/County,0.221961,0.709697,0.12,0.012424,73,99.0
+1,Sister Cities,0.241569,0.725152,0.102745,0.013939,74,99.0
+1,it is up to the,0.252941,0.741818,0.100392,0.013939,75,98.0
+1,International will,0.216078,0.75697,0.139608,0.015152,76,98.0
+1,cooperative agreement.,0.198431,0.774545,0.189804,0.013333,77,99.0
+1,relationship into,0.203137,0.796667,0.133725,0.015152,78,98.0
+1,write a letter,0.251765,0.813939,0.104706,0.014242,79,98.0
+1,"cities, but",0.230196,0.83,0.080392,0.013333,80,98.0
+1,able to sustain,0.236471,0.845758,0.12,0.013636,81,98.0
+1,writing by the,0.213725,0.861818,0.112157,0.014848,82,98.0
+1,AL,0.375686,0.066667,0.032941,0.009091,83,92.0
+1,locally.,0.358431,0.085152,0.051765,0.015455,84,98.0
+1,formed when,0.332941,0.209394,0.110196,0.015152,85,98.0
+1,appointed,0.328627,0.22697,0.083922,0.013333,86,99.0
+1,sign a formal,0.370588,0.242424,0.101569,0.013939,87,98.0
+1,relationship. Sister,0.321569,0.258485,0.17098,0.013636,88,99.0
+1,by one or both,0.335686,0.274848,0.122353,0.013939,89,99.0
+1,formally,0.366667,0.298485,0.071765,0.014545,90,99.0
+1,current on,0.335686,0.314545,0.09098,0.014545,91,99.0
+1,International,0.366667,0.330606,0.108627,0.014848,92,99.0
+1,formed,0.381176,0.347576,0.065098,0.013636,93,98.0
+1,Cities,0.36902,0.393939,0.047843,0.015758,94,97.0
+1,"City""",0.376078,0.410909,0.043922,0.014545,95,99.0
+1,formal documents,0.32902,0.427273,0.146667,0.013333,96,99.0
+1,relationships by,0.32902,0.443333,0.133725,0.014242,97,98.0
+1,Board of,0.375294,0.467576,0.07451,0.013333,98,99.0
+1,will recognize,0.377255,0.490606,0.111765,0.014242,99,97.0
+1,"community,",0.366667,0.50697,0.098824,0.014242,100,98.0
+1,community,0.383529,0.523333,0.089804,0.013939,101,98.0
+1,among all,0.362745,0.539091,0.079216,0.015455,102,99.0
+1,cooperative,0.318431,0.555758,0.114902,0.013939,103,98.0
+1,reasonable amount,0.325098,0.571515,0.157255,0.013636,104,99.0
+1,as a,0.383529,0.587879,0.038039,0.014545,105,99.0
+1,membership,0.329804,0.604848,0.105098,0.013333,106,99.0
+1,agreement must be,0.305882,0.627879,0.158824,0.014545,107,99.0
+1,of the requesting,0.34902,0.644848,0.148627,0.013333,108,99.0
+1,Executive/Governor,0.286667,0.660909,0.164314,0.013939,109,97.0
+1,the Mayor/County,0.353333,0.67697,0.153333,0.013939,110,99.0
+1,"input by, the",0.349804,0.693333,0.100392,0.013333,111,98.0
+1,Executive/Governorto,0.341961,0.709697,0.185098,0.012424,112,99.0
+1,International,0.344314,0.725152,0.131765,0.013939,113,99.0
+1,requesting,0.353333,0.741818,0.093333,0.013939,114,98.0
+1,"not, in any",0.355686,0.75697,0.085098,0.015152,115,98.0
+1,"Emeritus status,",0.336863,0.796667,0.135294,0.015152,116,98.0
+1,to the mayor,0.356471,0.813939,0.106275,0.014242,117,98.0
+1,understand that,0.310588,0.83,0.133333,0.013333,118,98.0
+1,an active,0.356471,0.845758,0.076863,0.013636,119,98.0
+1,mayor of the U.S.,0.325882,0.861818,0.141176,0.014848,120,98.0
+1,the mayor or,0.443137,0.209394,0.107843,0.015152,121,98.0
+1,official) from a,0.412549,0.22697,0.138824,0.013333,122,99.0
+1,agreement,0.472157,0.242424,0.084706,0.013939,123,98.0
+1,city agreements,0.492549,0.258485,0.110588,0.013636,124,99.0
+1,of the respective,0.458039,0.274848,0.162745,0.013939,125,99.0
+1,recognize only,0.438431,0.298485,0.120784,0.014545,126,99.0
+1,membership dues),0.426667,0.314545,0.154118,0.014545,127,99.0
+1,shall not,0.475294,0.330606,0.071373,0.014848,128,99.0
+1,by non-members.,0.446275,0.347576,0.138431,0.013636,129,98.0
+1,relationship is often,0.416863,0.393939,0.16549,0.015758,130,97.0
+1,agreement. Typically,0.42,0.410909,0.167059,0.014545,131,99.0
+1,that are,0.475686,0.427273,0.067843,0.013333,132,99.0
+1,members in,0.462745,0.443333,0.096863,0.014242,133,98.0
+1,Directors:,0.449804,0.467576,0.082353,0.013333,134,99.0
+1,a new sister,0.48902,0.490606,0.101176,0.014242,135,97.0
+1,even though,0.46549,0.50697,0.104706,0.014242,136,98.0
+1,and a,0.473333,0.523333,0.049412,0.013939,137,98.0
+1,involved communities,0.441961,0.539091,0.174902,0.015455,138,99.0
+1,"agreement is denied,",0.433333,0.555758,0.147843,0.013939,139,98.0
+1,"of time,",0.482353,0.571515,0.063529,0.013636,140,99.0
+1,friendship city,0.421569,0.587879,0.142745,0.014545,141,99.0
+1,directories.,0.434902,0.604848,0.093725,0.013333,142,99.0
+1,sent by the,0.464706,0.627879,0.094902,0.014545,143,99.0
+1,"community,",0.497647,0.644848,0.081176,0.013333,144,99.0
+1,of each of the,0.45098,0.660909,0.117647,0.013939,145,97.0
+1,sister cities,0.450196,0.693333,0.1,0.013333,146,98.0
+1,sign,0.527059,0.709697,0.037647,0.012424,147,99.0
+1,will help with,0.476078,0.725152,0.081961,0.013939,148,99.0
+1,community to,0.446667,0.741818,0.109804,0.013939,149,98.0
+1,"way, force a",0.440784,0.75697,0.106275,0.015152,150,98.0
+1,the mayor,0.472157,0.796667,0.086667,0.015152,151,98.0
+1,of the foreign,0.462745,0.813939,0.120784,0.014242,152,98.0
+1,the relationship,0.443922,0.83,0.129804,0.013333,153,98.0
+1,relationship. Sister,0.433333,0.845758,0.155294,0.013636,154,98.0
+1,city of the,0.467059,0.861818,0.087451,0.014848,155,98.0
+1,highest elected,0.55098,0.209394,0.126667,0.015152,156,98.0
+1,U.S. community,0.551373,0.22697,0.122745,0.013333,157,99.0
+1,on behalf of,0.556863,0.242424,0.116078,0.013939,158,98.0
+1,shall be,0.603137,0.258485,0.063529,0.013636,159,99.0
+1,communities.,0.620784,0.274848,0.084314,0.013939,160,99.0
+1,those relationships,0.559216,0.298485,0.15451,0.014545,161,99.0
+1,in its,0.580784,0.314545,0.042353,0.014545,162,99.0
+1,assert as invalid,0.546667,0.330606,0.138039,0.014848,163,99.0
+1,formed by,0.582353,0.393939,0.088627,0.015758,164,97.0
+1,Friendship,0.587059,0.410909,0.089412,0.014545,165,99.0
+1,signed. Sister,0.543529,0.427273,0.11098,0.013333,166,99.0
+1,its Membership,0.559608,0.443333,0.126275,0.014242,167,98.0
+1,cities,0.590196,0.490606,0.046275,0.014242,168,97.0
+1,another,0.570196,0.50697,0.067451,0.014242,169,98.0
+1,different U.S.,0.522745,0.523333,0.116471,0.013939,170,98.0
+1,is filed,0.616863,0.539091,0.060784,0.015455,171,99.0
+1,or no response,0.581176,0.555758,0.127059,0.013939,172,98.0
+1,Sister Cities,0.545882,0.571515,0.102353,0.013636,173,99.0
+1,and it will be,0.564314,0.587879,0.079608,0.014545,174,99.0
+1,Mayor/County,0.559608,0.627879,0.116863,0.014545,175,99.0
+1,and must be,0.578824,0.644848,0.100784,0.013333,176,99.0
+1,existing,0.568627,0.660909,0.064314,0.013939,177,97.0
+1,Executive/Governor,0.506667,0.67697,0.165882,0.013939,178,99.0
+1,"program, it is",0.550196,0.693333,0.11098,0.013333,179,98.0
+1,the cooperative,0.564706,0.709697,0.130196,0.012424,180,99.0
+1,the cooperative,0.558039,0.725152,0.131765,0.013939,181,99.0
+1,get the agreement,0.556471,0.741818,0.152941,0.013939,182,98.0
+1,community to,0.547059,0.75697,0.108627,0.015152,183,98.0
+1,or highest,0.558824,0.796667,0.086275,0.015152,184,98.0
+1,city indicating,0.583529,0.813939,0.114118,0.014242,185,98.0
+1,will remain,0.573725,0.83,0.088235,0.013333,186,98.0
+1,Cities,0.588627,0.845758,0.048235,0.013636,187,98.0
+1,situation. Sister,0.55451,0.861818,0.128235,0.014848,188,98.0
+1,"official (or,",0.677647,0.209394,0.103137,0.015152,189,98.0
+1,and a community,0.674118,0.22697,0.139608,0.013333,190,99.0
+1,their communities,0.672941,0.242424,0.114902,0.013939,191,98.0
+1,considered,0.666667,0.258485,0.092549,0.013636,192,99.0
+1,by,0.713725,0.298485,0.023529,0.014545,193,99.0
+1,Membership Directory,0.623137,0.314545,0.183529,0.014545,194,99.0
+1,or otherwise,0.684706,0.330606,0.135294,0.014848,195,99.0
+1,cities as a,0.67098,0.393939,0.085098,0.015758,196,97.0
+1,City agreements,0.676471,0.410909,0.133725,0.014545,197,99.0
+1,Cities International,0.65451,0.427273,0.155686,0.013333,198,99.0
+1,Directory,0.685882,0.443333,0.083137,0.014242,199,98.0
+1,affiliation between,0.636471,0.490606,0.151373,0.014242,200,97.0
+1,affiliation may exist,0.637647,0.50697,0.158824,0.014242,201,98.0
+1,"community, only",0.639216,0.523333,0.131373,0.013939,202,98.0
+1,with Sister,0.677647,0.539091,0.091373,0.015455,203,99.0
+1,to the,0.708235,0.555758,0.048627,0.013939,204,98.0
+1,International will,0.648235,0.571515,0.13451,0.013636,205,99.0
+1,delineated as such,0.643922,0.587879,0.147059,0.014545,206,99.0
+1,sent to the,0.679608,0.644848,0.085098,0.013333,207,99.0
+1,partnership,0.632941,0.660909,0.092941,0.013939,208,97.0
+1,may request,0.672549,0.67697,0.106275,0.013939,209,99.0
+1,up to the discretion,0.661176,0.693333,0.154902,0.013333,210,98.0
+1,agreement.,0.694902,0.709697,0.088235,0.012424,211,99.0
+1,agreement,0.689804,0.725152,0.085882,0.013939,212,99.0
+1,signed.,0.709412,0.741818,0.060784,0.013939,213,98.0
+1,"""share"" and sign",0.655686,0.75697,0.138039,0.015152,214,98.0
+1,elected official,0.645098,0.796667,0.128235,0.015152,215,98.0
+1,that they,0.697647,0.813939,0.075294,0.014242,216,98.0
+1,inactive until,0.661961,0.83,0.104706,0.013333,217,98.0
+1,International should,0.636863,0.845758,0.168235,0.013636,218,98.0
+1,Cities,0.682745,0.861818,0.048235,0.014848,219,98.0
+1,if elections,0.780784,0.209394,0.076471,0.015152,220,98.0
+1,in,0.813725,0.22697,0.013725,0.013333,221,99.0
+1,endorsing,0.787843,0.242424,0.083529,0.013939,222,98.0
+1,active/valid,0.759216,0.258485,0.091373,0.013636,223,99.0
+1,cities/members,0.737255,0.298485,0.127451,0.014545,224,99.0
+1,or on its,0.806667,0.314545,0.067059,0.014545,225,99.0
+1,impugn the,0.82,0.330606,0.057647,0.014848,226,99.0
+1,"""stepping",0.756078,0.393939,0.076471,0.015758,227,97.0
+1,are,0.810196,0.410909,0.028235,0.014545,228,99.0
+1,shall,0.810196,0.427273,0.035686,0.013333,229,99.0
+1,and website.,0.76902,0.443333,0.096863,0.014242,230,98.0
+1,a,0.787843,0.490606,0.009412,0.014242,231,97.0
+1,if a,0.770588,0.523333,0.028627,0.013939,232,98.0
+1,Cities,0.76902,0.539091,0.043922,0.015455,233,99.0
+1,request,0.756863,0.555758,0.061961,0.013939,234,98.0
+1,input,0.778824,0.67697,0.038824,0.013939,235,99.0
+1,Sister,0.770196,0.741818,0.049804,0.013939,236,98.0
+1,of the U.S.,0.773333,0.796667,0.081176,0.015152,237,98.0
+1,wish to,0.772941,0.813939,0.054902,0.014242,238,98.0
+1,such time as,0.766667,0.83,0.099608,0.013333,239,98.0
+1,be,0.805098,0.845758,0.018824,0.013636,240,98.0
+1,International will,0.73098,0.861818,0.134118,0.014848,241,98.0
+1,Toolkit,0.829412,0.069394,0.132549,0.031515,242,99.0
+1,a,0.871373,0.242424,0.00902,0.013939,243,98.0
+1,in,0.864706,0.298485,0.012941,0.014545,244,99.0
+2,SisterCities Partnership Agreement,0.167843,0.025455,0.793725,0.03697,1,99.0
+2,IN TE,0.17098,0.066667,0.072157,0.009091,2,87.0
+2,Connect,0.168627,0.085758,0.067451,0.014242,3,99.0
+2,then place the,0.116471,0.130909,0.115294,0.014242,4,98.0
+2,lists of sister,0.116471,0.147576,0.109804,0.014545,5,97.0
+2,If a community,0.116863,0.171515,0.12549,0.014242,6,98.0
+2,highest elected,0.116863,0.187273,0.124314,0.014242,7,98.0
+2,Cities International,0.116863,0.203636,0.159216,0.014242,8,97.0
+2,and Sister Cities,0.116863,0.220303,0.132549,0.012727,9,98.0
+2,of sister city,0.117647,0.237273,0.119216,0.012424,10,99.0
+2,dormant. Many,0.116471,0.252424,0.122745,0.013939,11,98.0
+2,partnership may,0.116471,0.267879,0.135294,0.015758,12,99.0
+2,General,0.117647,0.293636,0.106275,0.018182,13,99.0
+2,In order for a,0.117255,0.32303,0.119216,0.014848,14,98.0
+2,"(SCl), the two",0.118039,0.339394,0.115686,0.014242,15,98.0
+2,presumes,0.116471,0.355758,0.085882,0.014848,16,99.0
+2,followed proper,0.116078,0.371515,0.13451,0.015152,17,99.0
+2,with the specific,0.117255,0.388485,0.136471,0.013636,18,98.0
+2,and that both,0.116863,0.405152,0.108235,0.013333,19,98.0
+2,should check,0.116078,0.420303,0.10902,0.015152,20,98.0
+2,before pursuing,0.117255,0.437576,0.133725,0.013636,21,99.0
+2,SCl often refers,0.116078,0.459697,0.147843,0.014848,22,99.0
+2,"Understanding.""",0.116471,0.476364,0.137647,0.014848,23,99.0
+2,your documents,0.116863,0.493939,0.134902,0.013333,24,99.0
+2,A few things to,0.116471,0.516667,0.122353,0.014545,25,99.0
+2,Your,0.176078,0.541515,0.044314,0.013636,26,98.0
+2,commitment,0.174902,0.557273,0.101961,0.014242,27,98.0
+2,with,0.175294,0.573939,0.034118,0.013333,28,99.0
+2,related,0.176078,0.590303,0.063137,0.013333,29,99.0
+2,Don't try,0.176078,0.619394,0.072941,0.013939,30,99.0
+2,interest,0.175686,0.636061,0.068627,0.013333,31,99.0
+2,include,0.174902,0.651818,0.061569,0.013939,32,99.0
+2,the scope,0.175294,0.668788,0.081569,0.013333,33,98.0
+2,"tasks,",0.174902,0.684848,0.051765,0.01303,34,99.0
+2,administration,0.175294,0.700909,0.114902,0.013636,35,98.0
+2,memorandum,0.175294,0.71697,0.112549,0.013636,36,99.0
+2,agreement,0.175686,0.73303,0.092941,0.013939,37,98.0
+2,with very,0.175294,0.749697,0.076863,0.013333,38,98.0
+2,Work,0.175686,0.778485,0.047059,0.013636,39,99.0
+2,share,0.175686,0.795152,0.05098,0.013333,40,99.0
+2,what,0.175294,0.811212,0.041569,0.013939,41,99.0
+2,Ask your,0.175686,0.840606,0.075294,0.013636,42,98.0
+2,important,0.176078,0.85697,0.079608,0.013333,43,99.0
+2,the,0.174902,0.87303,0.03098,0.014242,44,99.0
+2,RN A TIO,0.243137,0.066667,0.108235,0.009091,45,87.0
+2,globally. Thrive,0.236078,0.085758,0.121569,0.014242,46,99.0
+2,partnership,0.231765,0.130909,0.100392,0.014242,47,98.0
+2,city programs.,0.226275,0.147576,0.111765,0.014545,48,97.0
+2,wishes to,0.242353,0.171515,0.078039,0.014242,49,98.0
+2,official of,0.241176,0.187273,0.106667,0.014242,50,98.0
+2,should,0.276078,0.203636,0.056863,0.014242,51,97.0
+2,International,0.249412,0.220303,0.10902,0.012727,52,98.0
+2,programs.,0.236863,0.237273,0.096471,0.012424,53,99.0
+2,partnerships,0.239216,0.252424,0.111373,0.013939,54,98.0
+2,be reinvigorated,0.251765,0.267879,0.136863,0.015758,55,99.0
+2,Guidelines,0.223922,0.293636,0.125882,0.018182,56,99.0
+2,sister,0.236471,0.32303,0.057255,0.014848,57,98.0
+2,communities,0.233725,0.339394,0.103529,0.014242,58,98.0
+2,several key,0.202353,0.355758,0.116863,0.014848,59,99.0
+2,procedures,0.250588,0.371515,0.128627,0.015152,60,99.0
+2,city);,0.253725,0.388485,0.082745,0.013636,61,98.0
+2,have secured,0.225098,0.405152,0.115294,0.013333,62,98.0
+2,with your local,0.225098,0.420303,0.120392,0.015152,63,98.0
+2,a sister city,0.25098,0.437576,0.098039,0.013636,64,99.0
+2,to these,0.263922,0.459697,0.080392,0.014848,65,99.0
+2,"However,",0.254118,0.476364,0.080392,0.014848,66,99.0
+2,is left up to,0.251765,0.493939,0.097255,0.013333,67,99.0
+2,keep in mind,0.238824,0.516667,0.106275,0.014545,68,99.0
+2,agreement can,0.220392,0.541515,0.124314,0.013636,69,98.0
+2,to fostering,0.276863,0.557273,0.099608,0.014242,70,98.0
+2,particular areas,0.209412,0.573939,0.134118,0.013333,71,99.0
+2,to anything,0.239216,0.590303,0.093333,0.013333,72,99.0
+2,to include,0.24902,0.619394,0.080784,0.013939,73,99.0
+2,or participating,0.244314,0.636061,0.13098,0.013333,74,99.0
+2,all the programs,0.236471,0.651818,0.139608,0.013939,75,99.0
+2,of projects.,0.256863,0.668788,0.098039,0.013333,76,98.0
+2,"responsibilities,",0.226667,0.684848,0.12902,0.01303,77,99.0
+2,of the,0.290196,0.700909,0.052549,0.013636,78,98.0
+2,between,0.287843,0.71697,0.074118,0.013636,79,99.0
+2,is a,0.268627,0.73303,0.033333,0.013939,80,98.0
+2,specific,0.252157,0.749697,0.068627,0.013333,81,98.0
+2,with your,0.222745,0.778485,0.080392,0.013636,82,99.0
+2,drafts of,0.226667,0.795152,0.106667,0.013333,83,99.0
+2,they'd like to see,0.216863,0.811212,0.139608,0.013939,84,99.0
+2,counterparts,0.25098,0.840606,0.108627,0.013636,85,98.0
+2,for the,0.255686,0.85697,0.062745,0.013333,86,99.0
+2,commitment their,0.205882,0.87303,0.146667,0.014242,87,99.0
+2,N A L,0.351373,0.066667,0.057255,0.009091,88,87.0
+2,locally.,0.357647,0.085758,0.051373,0.014242,89,99.0
+2,into Emeritus,0.332157,0.130909,0.108627,0.014242,90,98.0
+2,terminate a,0.320392,0.171515,0.096863,0.014242,91,98.0
+2,the U.S. city,0.347843,0.187273,0.076078,0.014242,92,98.0
+2,be informed,0.332941,0.203636,0.102745,0.014242,93,97.0
+2,will then,0.358431,0.220303,0.065882,0.012727,94,98.0
+2,We do not,0.333333,0.237273,0.06549,0.012424,95,99.0
+2,wax and,0.350588,0.252424,0.070196,0.013939,96,98.0
+2,by local,0.388627,0.267879,0.064314,0.015758,97,99.0
+2,city/county/state,0.293725,0.32303,0.144314,0.014848,98,98.0
+2,must sign,0.337255,0.339394,0.081569,0.014242,99,98.0
+2,items: that the,0.319216,0.355758,0.109412,0.014848,100,99.0
+2,(e.g. passed,0.379216,0.371515,0.073725,0.015152,101,99.0
+2,that both,0.336471,0.388485,0.066275,0.013636,102,98.0
+2,the necessary,0.340392,0.405152,0.120784,0.013333,103,98.0
+2,sister city,0.34549,0.420303,0.084706,0.015152,104,98.0
+2,relationship.,0.34902,0.437576,0.093725,0.013636,105,99.0
+2,agreements as,0.344314,0.459697,0.101569,0.014848,106,99.0
+2,as the following,0.33451,0.476364,0.133725,0.014848,107,99.0
+2,you.,0.34902,0.493939,0.031765,0.013333,108,99.0
+2,as you draft,0.345098,0.516667,0.101176,0.014545,109,99.0
+2,range from,0.344706,0.541515,0.117647,0.013636,110,98.0
+2,"of interest,",0.343529,0.573939,0.092157,0.013333,111,99.0
+2,from numbers,0.332549,0.590303,0.132549,0.013333,112,99.0
+2,everything,0.329804,0.619394,0.091373,0.013939,113,99.0
+2,institutions,0.375294,0.636061,0.109804,0.013333,114,99.0
+2,you plan,0.376078,0.651818,0.072941,0.013939,115,99.0
+2,This is a,0.354902,0.668788,0.070588,0.013333,116,98.0
+2,or other,0.355686,0.684848,0.072941,0.01303,117,99.0
+2,partnership,0.342745,0.700909,0.098824,0.013636,118,98.0
+2,the respective,0.361961,0.71697,0.118431,0.013636,119,99.0
+2,historical,0.301961,0.73303,0.10902,0.013939,120,98.0
+2,tasks.,0.320784,0.749697,0.045882,0.013333,121,98.0
+2,counterparts.,0.303137,0.778485,0.136471,0.013636,122,99.0
+2,your agreement,0.333333,0.795152,0.128235,0.013333,123,99.0
+2,in the,0.356471,0.811212,0.049412,0.013939,124,99.0
+2,to translate,0.359608,0.840606,0.094902,0.013636,125,98.0
+2,citizens of,0.318431,0.85697,0.119216,0.013333,126,99.0
+2,city has,0.352549,0.87303,0.07098,0.014242,127,99.0
+2,Status and,0.440784,0.130909,0.08902,0.014242,128,98.0
+2,sister city,0.417255,0.171515,0.082745,0.014242,129,98.0
+2,should be sent,0.423922,0.187273,0.120784,0.014242,130,98.0
+2,of this action,0.435686,0.203636,0.110588,0.014242,131,97.0
+2,remove the,0.424314,0.220303,0.094902,0.012727,132,98.0
+2,recommend,0.398824,0.237273,0.098039,0.012424,133,99.0
+2,wane over the,0.420784,0.252424,0.12,0.013939,134,98.0
+2,members,0.452941,0.267879,0.077255,0.015758,135,99.0
+2,partnership,0.438039,0.32303,0.087451,0.014848,136,98.0
+2,formal documents,0.418824,0.339394,0.153333,0.014242,137,98.0
+2,U.S. community,0.428627,0.355758,0.12,0.014848,138,99.0
+2,a city council,0.452941,0.371515,0.108627,0.015152,139,99.0
+2,communities share,0.402745,0.388485,0.137647,0.013636,140,98.0
+2,support,0.461176,0.405152,0.068235,0.013333,141,98.0
+2,program to,0.430196,0.420303,0.09451,0.015152,142,98.0
+2,"a ""Sister",0.445882,0.459697,0.100392,0.014848,143,99.0
+2,examples,0.468235,0.476364,0.081176,0.014848,144,99.0
+2,your agreement:,0.446275,0.516667,0.134902,0.014545,145,99.0
+2,"the ceremonial,",0.462353,0.541515,0.107451,0.013636,146,98.0
+2,"understanding,",0.376471,0.557273,0.125098,0.014242,147,98.0
+2,specific,0.435686,0.573939,0.067843,0.013333,148,99.0
+2,of,0.465098,0.590303,0.037255,0.013333,149,99.0
+2,you plan to do.,0.421176,0.619394,0.121569,0.013939,150,99.0
+2,are good,0.485098,0.636061,0.067451,0.013333,151,99.0
+2,to do if it,0.44902,0.651818,0.078431,0.013939,152,99.0
+2,formal document,0.42549,0.668788,0.139216,0.013333,153,98.0
+2,nuts-and-bolts,0.428627,0.684848,0.125882,0.01303,154,99.0
+2,can be,0.441569,0.700909,0.056471,0.013636,155,98.0
+2,sister,0.480392,0.71697,0.052157,0.013636,156,99.0
+2,document and should,0.41098,0.73303,0.149412,0.013939,157,98.0
+2,Remember that,0.439608,0.778485,0.108235,0.013636,158,99.0
+2,with your,0.461569,0.795152,0.05098,0.013333,159,99.0
+2,agreement. Be,0.405882,0.811212,0.122745,0.013939,160,99.0
+2,the agreement,0.45451,0.840606,0.116471,0.013636,161,98.0
+2,your partner,0.437647,0.85697,0.09098,0.013333,162,99.0
+2,made. Have,0.423529,0.87303,0.098039,0.014242,163,99.0
+2,will reflect,0.529804,0.130909,0.089412,0.014242,164,98.0
+2,"relationship, then",0.5,0.171515,0.141176,0.014242,165,98.0
+2,to the mayor,0.544706,0.187273,0.106667,0.014242,166,98.0
+2,in writing,0.546275,0.203636,0.076863,0.014242,167,97.0
+2,partnership,0.519216,0.220303,0.096863,0.012727,168,98.0
+2,terminating a,0.496863,0.237273,0.109804,0.012424,169,99.0
+2,"years, and",0.540784,0.252424,0.09098,0.013939,170,98.0
+2,years after,0.530196,0.267879,0.100392,0.015758,171,99.0
+2,to be recognized,0.52549,0.32303,0.12902,0.014848,172,98.0
+2,which,0.572157,0.339394,0.050196,0.014242,173,98.0
+2,is already,0.548627,0.355758,0.085882,0.014848,174,99.0
+2,resolution,0.561569,0.371515,0.08549,0.015152,175,99.0
+2,a mutual,0.540392,0.388485,0.067451,0.013636,176,98.0
+2,structure to,0.529412,0.405152,0.109804,0.013333,177,98.0
+2,see if they,0.524706,0.420303,0.09098,0.015152,178,98.0
+2,"City Agreement""",0.546275,0.459697,0.110196,0.014848,179,99.0
+2,"show, the",0.549412,0.476364,0.081569,0.014848,180,99.0
+2,with,0.569804,0.541515,0.035686,0.013636,181,98.0
+2,"cooperation, and",0.501569,0.557273,0.14,0.014242,182,98.0
+2,"programs/activities,",0.503529,0.573939,0.162745,0.013333,183,99.0
+2,exchanges to,0.502353,0.590303,0.114118,0.013333,184,99.0
+2,Some,0.542745,0.619394,0.052157,0.013939,185,99.0
+2,to include.,0.552549,0.636061,0.076078,0.013333,186,99.0
+2,makes the,0.527451,0.651818,0.087451,0.013939,187,99.0
+2,to establish,0.564706,0.668788,0.096471,0.013333,188,98.0
+2,text related,0.55451,0.684848,0.107059,0.01303,189,99.0
+2,expressed more,0.498039,0.700909,0.134902,0.013636,190,98.0
+2,city committees.,0.532549,0.71697,0.132941,0.013636,191,99.0
+2,not be,0.560392,0.73303,0.055294,0.013939,192,98.0
+2,this is signed,0.547843,0.778485,0.11098,0.013636,193,99.0
+2,international,0.512549,0.795152,0.101569,0.013333,194,99.0
+2,flexible to,0.528627,0.811212,0.082353,0.013939,195,99.0
+2,if it is,0.57098,0.840606,0.049412,0.013636,196,98.0
+2,community to,0.528627,0.85697,0.095294,0.013333,197,99.0
+2,someone in,0.521569,0.87303,0.098039,0.014242,198,99.0
+2,this status in,0.619216,0.130909,0.106275,0.014242,199,98.0
+2,a letter from,0.641176,0.171515,0.107059,0.014242,200,98.0
+2,of the,0.651373,0.187273,0.051373,0.014242,201,98.0
+2,by the mayor,0.623137,0.203636,0.109804,0.014242,202,97.0
+2,from its,0.616078,0.220303,0.068627,0.012727,203,98.0
+2,relationship,0.606667,0.237273,0.1,0.012424,204,99.0
+2,in many cases,0.631765,0.252424,0.118824,0.013939,205,98.0
+2,it has been,0.630588,0.267879,0.089804,0.015758,206,99.0
+2,by Sister,0.65451,0.32303,0.073725,0.014848,207,98.0
+2,clearly,0.622353,0.339394,0.061961,0.014242,208,98.0
+2,a member,0.63451,0.355758,0.084314,0.014848,209,99.0
+2,declaring,0.647059,0.371515,0.08,0.015152,210,99.0
+2,commitment to,0.607843,0.388485,0.118824,0.013636,211,98.0
+2,build a lasting,0.639216,0.405152,0.105882,0.013333,212,98.0
+2,have any,0.615686,0.420303,0.076471,0.015152,213,98.0
+2,or,0.656471,0.459697,0.053333,0.014848,214,99.0
+2,actual name,0.63098,0.476364,0.102745,0.014848,215,99.0
+2,language focusing,0.60549,0.541515,0.152549,0.013636,216,98.0
+2,mutual,0.641569,0.557273,0.057647,0.014242,217,98.0
+2,or more,0.666275,0.573939,0.067843,0.013333,218,99.0
+2,economic development.,0.616471,0.590303,0.158039,0.013333,219,99.0
+2,"specifics, like",0.594902,0.619394,0.112941,0.013939,220,99.0
+2,"However,",0.628627,0.636061,0.078431,0.013333,221,99.0
+2,document too,0.614902,0.651818,0.112941,0.013939,222,99.0
+2,the,0.661176,0.668788,0.030588,0.013333,223,98.0
+2,to,0.661569,0.684848,0.012941,0.01303,224,99.0
+2,fully in a,0.632941,0.700909,0.072549,0.013636,225,98.0
+2,Your,0.66549,0.71697,0.044314,0.013636,226,99.0
+2,dated or limited,0.615686,0.73303,0.130588,0.013939,227,98.0
+2,by both,0.658824,0.778485,0.065098,0.013636,228,99.0
+2,partners and,0.614118,0.795152,0.107843,0.013333,229,99.0
+2,cultural or,0.61098,0.811212,0.089412,0.013939,230,99.0
+2,drafted in,0.620392,0.840606,0.083529,0.013636,231,98.0
+2,be able to read,0.623922,0.85697,0.123529,0.013333,232,99.0
+2,your own,0.619608,0.87303,0.082745,0.014242,233,99.0
+2,directories,0.72549,0.130909,0.094118,0.014242,234,98.0
+2,the mayor,0.748235,0.171515,0.084706,0.014242,235,98.0
+2,sister city. Sister,0.702745,0.187273,0.137647,0.014242,236,98.0
+2,of the U.S.,0.732941,0.203636,0.08549,0.014242,237,97.0
+2,directories and all,0.684706,0.220303,0.147843,0.012727,238,98.0
+2,simply because,0.706667,0.237273,0.127059,0.012424,239,99.0
+2,a dormant,0.750588,0.252424,0.081961,0.013939,240,98.0
+2,inactive.,0.720392,0.267879,0.063529,0.015758,241,99.0
+2,Cities International,0.728235,0.32303,0.146667,0.014848,242,98.0
+2,endorse the link.,0.684314,0.339394,0.139608,0.014242,243,98.0
+2,of SCl and has,0.718824,0.355758,0.118039,0.014848,244,99.0
+2,the intent to,0.727059,0.371515,0.098824,0.015152,245,99.0
+2,the relationship;,0.726667,0.388485,0.132941,0.013636,246,98.0
+2,relationship.,0.745098,0.405152,0.101961,0.013333,247,98.0
+2,additional requirements,0.692157,0.420303,0.187059,0.015152,248,98.0
+2,"""Memorandum of",0.709804,0.459697,0.106667,0.014848,249,99.0
+2,and format,0.733725,0.476364,0.096471,0.014848,250,99.0
+2,on each,0.758039,0.541515,0.067843,0.013636,251,98.0
+2,benefit to the,0.699216,0.557273,0.113725,0.014242,252,98.0
+2,concrete,0.734118,0.573939,0.075294,0.013333,253,99.0
+2,particular areas,0.707843,0.619394,0.131373,0.013939,254,99.0
+2,there's no need,0.707059,0.636061,0.130196,0.013333,255,99.0
+2,lengthy or,0.727843,0.651818,0.088627,0.013939,256,99.0
+2,relationship;,0.691765,0.668788,0.104706,0.013333,257,98.0
+2,implementation or,0.67451,0.684848,0.137647,0.01303,258,99.0
+2,separate,0.70549,0.700909,0.071373,0.013636,259,98.0
+2,partnership,0.709804,0.71697,0.095294,0.013636,260,99.0
+2,by being,0.746275,0.73303,0.072941,0.013939,261,98.0
+2,cities. You,0.723922,0.778485,0.086275,0.013636,262,99.0
+2,solicit feedback,0.721961,0.795152,0.128627,0.013333,263,99.0
+2,municipal priorities.,0.700392,0.811212,0.156078,0.013939,264,99.0
+2,English. It is,0.703922,0.840606,0.095686,0.013636,265,98.0
+2,and understand,0.747451,0.85697,0.124314,0.013333,266,99.0
+2,community who,0.702353,0.87303,0.124314,0.014242,267,99.0
+2,and all,0.819608,0.130909,0.050588,0.014242,268,98.0
+2,or,0.832941,0.171515,0.018824,0.014242,269,98.0
+2,city,0.818431,0.203636,0.028627,0.014242,270,97.0
+2,lists,0.832549,0.220303,0.031765,0.012727,271,98.0
+2,it is,0.833725,0.237273,0.027843,0.012424,272,99.0
+2,This,0.823922,0.339394,0.034118,0.014242,273,98.0
+2,twin,0.825882,0.371515,0.030588,0.015152,274,99.0
+2,You,0.847059,0.405152,0.029412,0.013333,275,98.0
+2,of,0.830196,0.476364,0.019608,0.014848,276,99.0
+2,city's,0.825882,0.541515,0.041176,0.013636,277,98.0
+2,"precise,",0.812941,0.557273,0.06549,0.014242,278,98.0
+2,goals,0.809412,0.573939,0.040784,0.013333,279,99.0
+2,of,0.839216,0.619394,0.019216,0.013939,280,99.0
+2,to,0.837255,0.636061,0.015294,0.013333,281,99.0
+2,limits,0.816471,0.651818,0.04,0.013939,282,99.0
+2,specific,0.796471,0.668788,0.064706,0.013333,283,98.0
+2,aligned,0.819216,0.73303,0.057255,0.013939,284,98.0
+2,should,0.810196,0.778485,0.052941,0.013636,285,99.0
+2,on,0.850588,0.795152,0.018431,0.013333,286,99.0
+2,Toolkit,0.82902,0.069394,0.132941,0.031515,287,99.0
+3,SisterCities Partnership Agreement,0.167843,0.025758,0.793725,0.03697,1,99.0
+3,akaplan@sister-cities.org,0.442745,0.781818,0.206275,0.014848,2,98.0
+3,INTERN,0.17098,0.066667,0.110196,0.009091,3,93.0
+3,Connect,0.168627,0.085455,0.067451,0.014848,4,99.0
+3,speaks,0.176078,0.131818,0.061569,0.013636,5,97.0
+3,you have,0.175294,0.148788,0.078431,0.013333,6,97.0
+3,Keep it to,0.175686,0.17697,0.080392,0.014242,7,99.0
+3,work best,0.174902,0.193636,0.083529,0.014242,8,99.0
+3,Most sister,0.175294,0.222727,0.096078,0.013939,9,99.0
+3,of the,0.174902,0.239091,0.054902,0.014242,10,99.0
+3,and,0.176078,0.256364,0.03451,0.012727,11,97.0
+3,Consider,0.175686,0.284545,0.075686,0.014242,12,98.0
+3,logos to,0.176078,0.301212,0.066667,0.013939,13,99.0
+3,city hall or,0.175294,0.317576,0.087451,0.013939,14,99.0
+3,Look at,0.17451,0.346364,0.064706,0.014848,15,98.0
+3,of what is,0.174902,0.362727,0.083137,0.014545,16,99.0
+3,cannot,0.174118,0.378485,0.058824,0.015758,17,99.0
+3,have them,0.174902,0.395758,0.086275,0.014242,18,98.0
+3,Documents,0.175294,0.424848,0.093725,0.014545,19,98.0
+3,Check with,0.174902,0.453939,0.091373,0.015455,20,98.0
+3,is OK with,0.17451,0.470606,0.083137,0.014545,21,98.0
+3,don't want,0.175686,0.487576,0.087059,0.013636,22,98.0
+3,Official,0.175294,0.516364,0.064706,0.014242,23,98.0
+3,partnership.,0.175294,0.533333,0.103922,0.013333,24,99.0
+3,for their,0.174902,0.549394,0.069804,0.012424,25,99.0
+3,Remember,0.175294,0.578485,0.089804,0.013636,26,98.0
+3,receive,0.175686,0.595152,0.065882,0.013333,27,99.0
+3,it is included,0.17451,0.610303,0.104314,0.015455,28,98.0
+3,Remember that,0.116863,0.639091,0.125882,0.015152,29,99.0
+3,like the establishment,0.116863,0.655152,0.178431,0.015455,30,98.0
+3,"others, before",0.116863,0.672424,0.122745,0.013939,31,99.0
+3,office to see if,0.117255,0.689091,0.146667,0.012727,32,99.0
+3,On the following,0.116471,0.716364,0.131373,0.015152,33,99.0
+3,what is possible.,0.116471,0.733636,0.13451,0.013939,34,99.0
+3,encourage you to,0.116471,0.749394,0.145098,0.014545,35,98.0
+3,you are unsure,0.115686,0.766061,0.129412,0.013939,36,99.0
+3,sending it to our,0.115686,0.781818,0.132941,0.014848,37,98.0
+3,347-8630.,0.116078,0.798182,0.083529,0.012727,38,99.0
+3,ATION,0.281176,0.066667,0.093333,0.009091,39,93.0
+3,globally. Thrive,0.236078,0.085455,0.121569,0.014848,40,99.0
+3,that language,0.237647,0.131818,0.110196,0.013636,41,97.0
+3,in your own,0.253725,0.148788,0.098824,0.013333,42,97.0
+3,one page.,0.256078,0.17697,0.081961,0.014242,43,99.0
+3,if they can,0.258431,0.193636,0.090196,0.014242,44,99.0
+3,city,0.271373,0.222727,0.034118,0.013939,45,99.0
+3,sister city,0.229804,0.239091,0.088235,0.014242,46,99.0
+3,cooperation.,0.210588,0.256364,0.101569,0.012727,47,97.0
+3,using official,0.251373,0.284545,0.109804,0.014242,48,98.0
+3,reflect your,0.242745,0.301212,0.112157,0.013939,49,99.0
+3,other municipal,0.262745,0.317576,0.123922,0.013939,50,99.0
+3,other agreements,0.239216,0.346364,0.150196,0.014848,51,98.0
+3,acceptable,0.258039,0.362727,0.09098,0.014545,52,99.0
+3,access older,0.232941,0.378485,0.109804,0.015758,53,99.0
+3,"on file,",0.261176,0.395758,0.059608,0.014242,54,98.0
+3,must be,0.26902,0.424848,0.069804,0.014545,55,98.0
+3,"your mayor,",0.266275,0.453939,0.104314,0.015455,56,98.0
+3,them. The,0.257647,0.470606,0.085882,0.014545,57,98.0
+3,to spend,0.262745,0.487576,0.076078,0.013636,58,98.0
+3,documents are,0.24,0.516364,0.122745,0.014242,59,98.0
+3,Be sure,0.279216,0.533333,0.068627,0.013333,60,99.0
+3,records.,0.244706,0.549394,0.063922,0.012424,61,99.0
+3,to send your,0.265098,0.578485,0.10549,0.013636,62,98.0
+3,your agreement,0.241569,0.595152,0.133725,0.013333,63,99.0
+3,in our,0.278824,0.610303,0.052941,0.015455,64,98.0
+3,each city's,0.242745,0.639091,0.090588,0.015152,65,99.0
+3,of a,0.295294,0.655152,0.039216,0.015455,66,98.0
+3,sanctioning a,0.239608,0.672424,0.10902,0.013939,67,99.0
+3,this is the case.,0.263922,0.689091,0.094902,0.012727,68,99.0
+3,pages you'll,0.247843,0.716364,0.101569,0.015152,69,99.0
+3,While you,0.25098,0.733636,0.083529,0.013939,70,99.0
+3,make your,0.261569,0.749394,0.089412,0.014545,71,98.0
+3,about your,0.245098,0.766061,0.091765,0.013939,72,99.0
+3,Membership,0.248627,0.781818,0.104706,0.014848,73,98.0
+3,AL,0.37451,0.066667,0.034118,0.009091,74,93.0
+3,locally.,0.357647,0.085455,0.051373,0.014848,75,99.0
+3,check the,0.347843,0.131818,0.083529,0.013636,76,97.0
+3,agreement.,0.352549,0.148788,0.090196,0.013333,77,97.0
+3,Ceremonial,0.338039,0.17697,0.096863,0.014242,78,99.0
+3,be posted in,0.348627,0.193636,0.1,0.014242,79,99.0
+3,agreements include,0.30549,0.222727,0.162353,0.013939,80,99.0
+3,movement-to,0.318039,0.239091,0.11098,0.014242,81,99.0
+3,letterhead,0.361176,0.284545,0.086275,0.014242,82,98.0
+3,enhance the,0.354902,0.301212,0.094118,0.013939,83,99.0
+3,offices,0.386667,0.317576,0.063922,0.013939,84,99.0
+3,your city,0.389412,0.346364,0.077647,0.014848,85,98.0
+3,"or possible,",0.34902,0.362727,0.096863,0.014545,86,99.0
+3,agreements,0.342745,0.378485,0.100392,0.015758,87,99.0
+3,although we do,0.320784,0.395758,0.126667,0.014242,88,98.0
+3,signed by the,0.338824,0.424848,0.113333,0.014545,89,98.0
+3,"city council,",0.370588,0.453939,0.099608,0.015455,90,98.0
+3,mayor is the,0.343529,0.470606,0.103529,0.014545,91,98.0
+3,time developing,0.338824,0.487576,0.131373,0.013636,92,98.0
+3,usually signed,0.362745,0.516364,0.116471,0.014242,93,98.0
+3,both communities,0.347843,0.533333,0.14549,0.013333,94,99.0
+3,signed,0.370588,0.578485,0.057647,0.013636,95,98.0
+3,we will post,0.375294,0.595152,0.092941,0.013333,96,99.0
+3,Annual Membership,0.331765,0.610303,0.161569,0.015455,97,98.0
+3,sister city,0.333333,0.639091,0.087059,0.015152,98,99.0
+3,"committee, a",0.33451,0.655152,0.109804,0.015455,99,98.0
+3,sister city,0.348627,0.672424,0.106275,0.013939,100,99.0
+3,find a series,0.349412,0.716364,0.102353,0.015152,101,99.0
+3,should feel free,0.33451,0.733636,0.13451,0.013939,102,99.0
+3,agreement,0.35098,0.749394,0.090196,0.014545,103,98.0
+3,agreement or,0.336863,0.766061,0.135686,0.013939,104,99.0
+3,Director at,0.353333,0.781818,0.089412,0.014848,105,98.0
+3,foreign-language,0.431373,0.131818,0.142745,0.013636,106,97.0
+3,documents such,0.434902,0.17697,0.138824,0.014242,107,99.0
+3,their entirety.,0.448627,0.193636,0.108235,0.014242,108,99.0
+3,some,0.467843,0.222727,0.046667,0.013939,109,99.0
+3,promote peace,0.42902,0.239091,0.126667,0.014242,110,99.0
+3,and/or other,0.447451,0.284545,0.11451,0.014242,111,98.0
+3,document.,0.44902,0.301212,0.083137,0.013939,112,99.0
+3,and should,0.450588,0.317576,0.088627,0.013939,113,99.0
+3,has signed.,0.467059,0.346364,0.094118,0.014848,114,98.0
+3,and they may,0.445882,0.362727,0.113725,0.014545,115,99.0
+3,please contact,0.443137,0.378485,0.121569,0.015758,116,99.0
+3,not have copies,0.447451,0.395758,0.132157,0.014242,117,98.0
+3,top elected,0.452157,0.424848,0.094902,0.014545,118,98.0
+3,"town clerk,",0.470196,0.453939,0.093333,0.015455,119,98.0
+3,one putting,0.447059,0.470606,0.095686,0.014545,120,98.0
+3,an agreement,0.470196,0.487576,0.119608,0.013636,121,98.0
+3,during a,0.479216,0.516364,0.070588,0.014242,122,98.0
+3,receive,0.493333,0.533333,0.062745,0.013333,123,99.0
+3,agreement to,0.428235,0.578485,0.112549,0.013636,124,98.0
+3,the relationship,0.468235,0.595152,0.128627,0.013333,125,99.0
+3,Directory.,0.493333,0.610303,0.081961,0.015455,126,98.0
+3,program is,0.420392,0.639091,0.09451,0.015152,127,99.0
+3,"review period,",0.438039,0.655152,0.120392,0.015455,128,98.0
+3,agreement. Check,0.454902,0.672424,0.12549,0.013939,129,99.0
+3,of partnership,0.451765,0.716364,0.124706,0.015152,130,99.0
+3,to use some,0.46902,0.733636,0.101569,0.013939,131,99.0
+3,your own and,0.441176,0.749394,0.112549,0.014545,132,98.0
+3,want advice,0.472549,0.766061,0.076471,0.013939,133,99.0
+3,version to,0.574118,0.131818,0.08549,0.013636,134,97.0
+3,as these,0.573725,0.17697,0.073333,0.014242,135,99.0
+3,acknowledgement,0.51451,0.222727,0.148627,0.013939,136,99.0
+3,through mutual,0.555686,0.239091,0.126667,0.014242,137,99.0
+3,embellishments,0.561961,0.284545,0.117255,0.014242,138,98.0
+3,Sister city,0.532157,0.301212,0.086667,0.013939,139,99.0
+3,reflect their,0.539216,0.317576,0.103137,0.013939,140,99.0
+3,These,0.561176,0.346364,0.05451,0.014848,141,98.0
+3,be in an easily,0.559608,0.362727,0.118824,0.014545,142,99.0
+3,Sister Cities,0.564706,0.378485,0.102353,0.015758,143,99.0
+3,of all,0.579608,0.395758,0.043137,0.014242,144,98.0
+3,official of,0.547059,0.424848,0.119608,0.014545,145,98.0
+3,et al. to make,0.563529,0.453939,0.111373,0.015455,146,98.0
+3,his or her name,0.542745,0.470606,0.13451,0.014545,147,98.0
+3,which will,0.589804,0.487576,0.08,0.013636,148,98.0
+3,formal ceremony,0.549804,0.516364,0.142745,0.014242,149,98.0
+3,a signed set,0.556078,0.533333,0.103137,0.013333,150,99.0
+3,Sister Cities,0.540784,0.578485,0.100784,0.013636,151,98.0
+3,in the,0.596863,0.595152,0.048235,0.013333,152,99.0
+3,independent and,0.514902,0.639091,0.136078,0.015152,153,99.0
+3,sustainability/funding,0.558431,0.655152,0.183529,0.015455,154,98.0
+3,with your,0.580392,0.672424,0.078431,0.013939,155,99.0
+3,agreements,0.576471,0.716364,0.1,0.015152,156,99.0
+3,of the,0.570588,0.733636,0.05451,0.013939,157,99.0
+3,be creative,0.553725,0.749394,0.096471,0.014545,158,98.0
+3,you can always,0.54902,0.766061,0.12549,0.013939,159,99.0
+3,make sure it,0.659608,0.131818,0.102353,0.013636,160,97.0
+3,partnership,0.647059,0.17697,0.099608,0.014242,161,99.0
+3,of the founding,0.663137,0.222727,0.128627,0.013939,162,99.0
+3,"respect,",0.682353,0.239091,0.072549,0.014242,163,99.0
+3,such as,0.679216,0.284545,0.066275,0.014242,164,98.0
+3,agreements are,0.618824,0.301212,0.132941,0.013939,165,99.0
+3,historical,0.642353,0.317576,0.078039,0.013939,166,99.0
+3,agreements may,0.615686,0.346364,0.137647,0.014848,167,98.0
+3,replicable,0.678431,0.362727,0.082745,0.014545,168,99.0
+3,"International,",0.667059,0.378485,0.114902,0.015758,169,99.0
+3,partnership,0.622745,0.395758,0.101961,0.014242,170,98.0
+3,both communities.,0.666667,0.424848,0.112941,0.014545,171,98.0
+3,sure that,0.674902,0.453939,0.079608,0.015455,172,98.0
+3,on the,0.677255,0.470606,0.055294,0.014545,173,98.0
+3,never be,0.669804,0.487576,0.080392,0.013636,174,98.0
+3,recognizing,0.692549,0.516364,0.098039,0.014242,175,98.0
+3,of the official,0.659216,0.533333,0.116471,0.013333,176,99.0
+3,International.,0.641569,0.578485,0.11098,0.013636,177,98.0
+3,City Directory,0.645098,0.595152,0.116863,0.013333,178,99.0
+3,can impose,0.65098,0.639091,0.09451,0.015152,179,99.0
+3,local program,0.658824,0.672424,0.115294,0.013939,180,99.0
+3,to give you,0.676471,0.716364,0.09098,0.015152,181,99.0
+3,formatting and,0.625098,0.733636,0.125882,0.013939,182,99.0
+3,with what you,0.650196,0.749394,0.111765,0.014545,183,98.0
+3,solicit,0.67451,0.766061,0.05098,0.013939,184,99.0
+3,or contacting,0.64902,0.781818,0.110196,0.014848,185,98.0
+3,mirrors what,0.761961,0.131818,0.104314,0.013636,186,97.0
+3,agreements,0.746667,0.17697,0.096863,0.014242,187,99.0
+3,principles,0.791765,0.222727,0.079608,0.013939,188,99.0
+3,"understanding,",0.754902,0.239091,0.120784,0.014242,189,99.0
+3,city seals or,0.74549,0.284545,0.098431,0.014242,190,98.0
+3,often posted,0.751765,0.301212,0.107059,0.013939,191,99.0
+3,importance,0.720392,0.317576,0.087451,0.013939,192,99.0
+3,give you an,0.753333,0.346364,0.095686,0.014848,193,98.0
+3,format. If,0.761176,0.362727,0.083137,0.014545,194,99.0
+3,we may,0.781961,0.378485,0.057647,0.015758,195,99.0
+3,agreements.,0.724706,0.395758,0.099216,0.014242,196,98.0
+3,the agreement,0.75451,0.453939,0.118039,0.015455,197,98.0
+3,"paper, and you",0.732549,0.470606,0.121961,0.014545,198,98.0
+3,signed.,0.750196,0.487576,0.056863,0.013636,199,98.0
+3,the,0.790588,0.516364,0.024706,0.014242,200,98.0
+3,documents,0.775686,0.533333,0.085882,0.013333,201,99.0
+3,After we,0.752549,0.578485,0.070588,0.013636,202,98.0
+3,and make,0.761961,0.595152,0.08,0.013333,203,99.0
+3,requirements,0.74549,0.639091,0.109804,0.015152,204,99.0
+3,"plan, among",0.741961,0.655152,0.092941,0.015455,205,98.0
+3,or mayor's,0.774118,0.672424,0.086667,0.013939,206,99.0
+3,an idea of,0.767451,0.716364,0.080392,0.015152,207,99.0
+3,"language, we",0.75098,0.733636,0.101176,0.013939,208,99.0
+3,produce.,0.761961,0.749394,0.094118,0.014545,209,98.0
+3,feedback by,0.72549,0.766061,0.100392,0.013939,210,99.0
+3,us at (202),0.759216,0.781818,0.089412,0.014848,211,98.0
+3,Toolkit,0.82902,0.069394,0.132941,0.031515,212,99.0
+3,at,0.858824,0.301212,0.014902,0.013939,213,99.0
+3,idea,0.84902,0.346364,0.032941,0.014848,214,98.0
+3,you,0.844314,0.362727,0.028627,0.014545,215,99.0
+3,sure,0.841961,0.595152,0.038039,0.013333,216,99.0
+3,If,0.856078,0.749394,0.007451,0.014545,217,98.0
+4,SisterCities Partnership Agreement,0.167843,0.025758,0.793725,0.03697,1,98.0
+4,ESTABLISHMENT,0.549412,0.371818,0.207451,0.015758,2,98.0
+4,IN TERN ATION,0.17098,0.066667,0.198824,0.009091,3,91.0
+4,Connect globally. Thrive,0.167451,0.084848,0.191373,0.016364,4,99.0
+4,AN,0.26,0.371818,0.037255,0.015758,5,98.0
+4,The Sister City,0.221176,0.523333,0.114118,0.014242,6,99.0
+4,By the President,0.221569,0.537273,0.123922,0.014545,7,99.0
+4,Friendship and,0.223137,0.552424,0.114902,0.012727,8,99.0
+4,Direct personal,0.222353,0.565758,0.115686,0.013939,9,99.0
+4,In order to,0.222745,0.59303,0.087451,0.012424,10,99.0
+4,Communities Friendship and by,0.222353,0.608485,0.121176,0.025455,11,98.0
+4,Abu Dhabi and,0.222745,0.649091,0.111765,0.010606,12,99.0
+4,"medicine, and",0.222745,0.66303,0.101176,0.011515,13,99.0
+4,hereby I proclaim,0.222745,0.676667,0.116078,0.012727,14,97.0
+4,the date of Houston,0.222745,0.69,0.139216,0.011818,15,98.0
+4,relationship,0.222745,0.704848,0.083137,0.011515,16,99.0
+4,Signed on this,0.222745,0.731818,0.109804,0.011818,17,99.0
+4,"Languages, both",0.222353,0.746061,0.124314,0.012121,18,99.0
+4,Sheikh,0.245882,0.806667,0.064314,0.010909,19,99.0
+4,Chairman,0.247059,0.823636,0.087451,0.010909,20,99.0
+4,A L,0.369804,0.066667,0.038824,0.009091,21,91.0
+4,locally.,0.358824,0.084848,0.051765,0.016364,22,99.0
+4,a,0.377647,0.291515,0.046275,0.018788,23,23.0
+4,ABU DHABI,0.375686,0.315455,0.082353,0.018182,24,99.0
+4,AGREEMENT,0.297255,0.371818,0.147451,0.015758,25,98.0
+4,SISTER,0.337647,0.392424,0.088627,0.013939,26,99.0
+4,THE CITY,0.336078,0.430303,0.116863,0.016667,27,98.0
+4,"HOUSTON,",0.385098,0.469697,0.127451,0.015455,28,96.0
+4,"Program,",0.335294,0.523333,0.072941,0.014242,29,99.0
+4,of the United,0.34549,0.537273,0.097255,0.014545,30,99.0
+4,understanding,0.338039,0.552424,0.106667,0.012727,31,99.0
+4,contact; and,0.338039,0.565758,0.091765,0.013939,32,99.0
+4,"foster those goals,",0.310196,0.59303,0.151765,0.012424,33,99.0
+4,"goodwill, agree",0.341569,0.606061,0.109412,0.013939,34,98.0
+4,"exploring education,",0.343529,0.620303,0.15098,0.013636,35,98.0
+4,"Houston, sharing",0.338039,0.648788,0.128235,0.011818,36,98.0
+4,the desire to promote,0.327843,0.66303,0.153725,0.011212,37,98.0
+4,themselves Sister,0.343922,0.677273,0.123529,0.009697,38,99.0
+4,City effective. Council,0.368627,0.690909,0.092157,0.024242,39,99.0
+4,became,0.311765,0.705758,0.05451,0.009091,40,99.0
+4,26 of October,0.332549,0.731818,0.103137,0.011818,41,98.0
+4,text being equally,0.346667,0.746061,0.132549,0.012121,42,99.0
+4,Mohammed bin,0.318039,0.806667,0.143922,0.01,43,99.0
+4,of Abu Dhabi,0.343137,0.823636,0.122353,0.010303,44,99.0
+4,&Town Planning,0.326275,0.841818,0.152549,0.010606,45,98.0
+4,4,0.502745,0.256364,0.085882,0.024848,46,33.0
+4,Labiig G,0.427059,0.29,0.137647,0.023636,47,66.0
+4,MUNICIPALITY,0.460392,0.316364,0.103137,0.017273,48,99.0
+4,FOR THE,0.444706,0.371818,0.104706,0.015758,49,98.0
+4,CITIES,0.426275,0.392424,0.08902,0.013939,50,99.0
+4,BETWEEN,0.45451,0.412121,0.111373,0.013333,51,99.0
+4,OF ABU,0.452941,0.430303,0.087451,0.016667,52,98.0
+4,AND,0.488627,0.451818,0.049412,0.012727,53,99.0
+4,TEXAS,0.512549,0.469697,0.081569,0.015455,54,96.0
+4,administered by Sister,0.408235,0.523333,0.172157,0.014242,55,99.0
+4,States of America,0.442745,0.537273,0.135294,0.014545,56,99.0
+4,between the United,0.446667,0.552121,0.141176,0.011515,57,99.0
+4,the people of,0.461961,0.59303,0.103922,0.012424,58,99.0
+4,to collaborate,0.45098,0.606061,0.102745,0.01303,59,98.0
+4,economic,0.49451,0.620303,0.074118,0.013636,60,98.0
+4,common,0.481176,0.648788,0.061569,0.010606,61,99.0
+4,mutual,0.487059,0.663333,0.057255,0.009394,62,99.0
+4,resolution Cities,0.466667,0.67697,0.077255,0.024545,63,99.0
+4,"2002, in duplicate",0.435686,0.732121,0.132549,0.010303,64,96.0
+4,authentic.,0.479216,0.746061,0.070196,0.012121,65,99.0
+4,Butti Al,0.468627,0.805455,0.076471,0.011212,66,98.0
+4,Municipality,0.471373,0.823333,0.115686,0.011515,67,99.0
+4,gi,0.560784,0.288788,0.03098,0.023636,68,49.0
+4,& TOWN PLANNING,0.567451,0.316061,0.138431,0.018788,69,79.0
+4,RELATIONSHIP,0.515294,0.392424,0.165098,0.013939,70,99.0
+4,DHABI ( U.,0.540392,0.430303,0.126667,0.016667,71,98.0
+4,( U.S.A),0.594118,0.469697,0.089804,0.015455,72,96.0
+4,Cities,0.580392,0.523333,0.044706,0.014242,73,99.0
+4,in 1956 to,0.578039,0.537273,0.071373,0.014545,74,99.0
+4,States and,0.587843,0.552121,0.081961,0.011515,75,99.0
+4,Abu Dhabi and,0.565882,0.59303,0.088627,0.012424,76,99.0
+4,for the mutual,0.553725,0.606061,0.105882,0.01303,77,98.0
+4,and cultural,0.568627,0.620303,0.093725,0.013636,78,98.0
+4,interest in energy,0.547451,0.648485,0.134118,0.011515,79,98.0
+4,understanding,0.541176,0.662121,0.110196,0.012121,80,98.0
+4,estatblishing the,0.543922,0.689394,0.121961,0.012121,81,99.0
+4,in the Arabic,0.568235,0.732121,0.1,0.010303,82,96.0
+4,Hamed,0.545098,0.805455,0.066667,0.011212,83,98.0
+4,OF,0.756863,0.371818,0.030196,0.015758,84,98.0
+4,A.E),0.667059,0.430303,0.047059,0.016667,85,98.0
+4,"International, was",0.625098,0.523333,0.139216,0.014242,86,99.0
+4,encourage greater,0.649412,0.537273,0.136863,0.014545,87,99.0
+4,other nations,0.669804,0.552121,0.102745,0.011515,88,99.0
+4,"Houston, in a",0.65451,0.59303,0.09451,0.012424,89,99.0
+4,benefit of their,0.659608,0.606061,0.11098,0.01303,90,98.0
+4,opportunities.,0.662353,0.620303,0.101569,0.013636,91,98.0
+4,", technology",0.681569,0.648485,0.09098,0.011515,92,98.0
+4,among our citizens,0.651373,0.662121,0.143137,0.012121,93,98.0
+4,Sister City,0.665882,0.689394,0.075294,0.012121,94,99.0
+4,and English,0.668235,0.732121,0.085098,0.010303,95,96.0
+4,fe,0.700784,0.752424,0.075686,0.045455,96,57.0
+4,Lee,0.728627,0.804545,0.038824,0.012121,97,99.0
+4,Mayor,0.70549,0.821818,0.06,0.013939,98,96.0
+4,initiated,0.764314,0.523333,0.056863,0.014242,99,99.0
+4,through,0.772549,0.552121,0.058039,0.011515,100,99.0
+4,gesture of,0.74902,0.59303,0.077647,0.012424,101,99.0
+4,and,0.772549,0.648485,0.024314,0.011515,102,98.0
+4,do,0.79451,0.662121,0.017647,0.012121,103,98.0
+4,P.Brown,0.768627,0.804848,0.081176,0.012424,104,99.0
+4,ofI Houston,0.76549,0.821818,0.105882,0.013939,105,98.0
+4,Toolkit,0.82902,0.069091,0.133333,0.032424,106,99.0
+5,SisterCities Partnership Agreement,0.167843,0.025758,0.793725,0.03697,1,99.0
+5,IN TERNAT,0.170588,0.066667,0.152549,0.009091,2,93.0
+5,Connect globally. Thrive,0.167451,0.084848,0.191373,0.016364,3,99.0
+5,THE,0.267451,0.353939,0.044706,0.014545,4,98.0
+5,The Sister City,0.2,0.402121,0.122745,0.014242,5,99.0
+5,beneficial solutions,0.202353,0.419091,0.154118,0.010303,6,99.0
+5,"Consequently, the",0.202745,0.434848,0.143137,0.011515,7,95.0
+5,important in their,0.201176,0.448788,0.139216,0.014242,8,99.0
+5,Encourage,0.229412,0.480303,0.088627,0.015152,9,99.0
+5,that they can,0.230588,0.49697,0.103529,0.013636,10,99.0
+5,Supporta and,0.230196,0.513333,0.098039,0.015455,11,98.0
+5,community,0.229412,0.527576,0.095294,0.015152,12,99.0
+5,medium for,0.229804,0.543636,0.09451,0.012727,13,99.0
+5,Generate an,0.23098,0.562121,0.094902,0.010606,14,99.0
+5,"by serving themes, as",0.22902,0.576667,0.132157,0.028182,15,99.0
+5,Identify,0.229804,0.593333,0.06549,0.012727,16,99.0
+5,nurture the i,0.228627,0.609091,0.096471,0.011818,17,92.0
+5,Promote the cities; key,0.228235,0.623333,0.106667,0.031515,18,98.0
+5,Provide financial,0.228627,0.655455,0.137647,0.014242,19,98.0
+5,aims of the,0.228235,0.670303,0.099216,0.015758,20,99.0
+5,With the above,0.198824,0.700303,0.122353,0.016364,21,98.0
+5,London soleinnly,0.198431,0.716667,0.141176,0.015455,22,98.0
+5,protocol of this,0.197647,0.731818,0.130588,0.017273,23,99.0
+5,This,0.308627,0.777576,0.037255,0.018485,24,98.0
+5,Rudolph,0.256078,0.874848,0.08,0.018485,25,98.0
+5,New,0.284314,0.906061,0.042353,0.017576,26,99.0
+5,IONAL,0.323137,0.066667,0.08549,0.009091,27,93.0
+5,locally.,0.358824,0.084848,0.051765,0.016364,28,99.0
+5,NEW YORK,0.312157,0.353939,0.114118,0.014545,29,98.0
+5,partnership,0.322745,0.402121,0.092549,0.014242,30,99.0
+5,to common,0.361176,0.419697,0.088627,0.008788,31,99.0
+5,Sister City,0.35098,0.433636,0.082745,0.011515,32,99.0
+5,network of,0.347451,0.449394,0.083922,0.010303,33,99.0
+5,and publicize,0.322745,0.481515,0.103922,0.012727,34,99.0
+5,flourish to,0.338431,0.496667,0.079608,0.010606,35,99.0
+5,promote the,0.33098,0.51303,0.097647,0.013939,36,99.0
+5,programs learning from to,0.326275,0.530606,0.108627,0.027576,37,99.0
+5,improvement,0.330196,0.559697,0.105882,0.013939,38,99.0
+5,a conduit of,0.338431,0.576364,0.098824,0.010606,39,99.0
+5,common,0.361176,0.590303,0.07098,0.014545,40,98.0
+5,increasingly,0.317647,0.606667,0.103137,0.014545,41,99.0
+5,mayoral,0.334902,0.63697,0.069804,0.017879,42,97.0
+5,or in kind,0.368235,0.655152,0.078431,0.012727,43,99.0
+5,Sister City,0.327451,0.670303,0.087843,0.015758,44,99.0
+5,purposes in,0.321176,0.700303,0.102745,0.016364,45,98.0
+5,confirm that,0.339608,0.716667,0.101176,0.015455,46,98.0
+5,Memorandum,0.328235,0.731818,0.110588,0.017273,47,99.0
+5,agreement,0.345882,0.777576,0.088627,0.018485,48,98.0
+5,huliaui,0.315294,0.807576,0.160392,0.073939,49,88.0
+5,W. Giuliani,0.336078,0.874848,0.089412,0.018485,50,98.0
+5,Mayor,0.30902,0.892121,0.06,0.015758,51,99.0
+5,York City,0.326667,0.906061,0.085882,0.017576,52,99.0
+5,The City,0.437255,0.259697,0.098824,0.013333,53,85.0
+5,OFFiCe,0.450588,0.276061,0.076471,0.011515,54,73.0
+5,"NEW YORK,",0.46,0.291212,0.105882,0.012121,55,89.0
+5,CITY-LONDON,0.426275,0.353939,0.147059,0.014545,56,98.0
+5,Memorandum,0.420784,0.369394,0.121961,0.014545,57,99.0
+5,between New,0.420784,0.401515,0.109412,0.01303,58,97.0
+5,challenges,0.456078,0.417879,0.082745,0.011515,59,99.0
+5,relationship,0.439608,0.43303,0.09098,0.012121,60,99.0
+5,global,0.437255,0.449091,0.049412,0.011818,61,99.0
+5,existing,0.431373,0.48,0.067059,0.013939,62,99.0
+5,benefit a wider,0.421569,0.496667,0.120784,0.010303,63,99.0
+5,development,0.430588,0.512121,0.10549,0.013939,64,99.0
+5,encourage both,0.423922,0.528485,0.122353,0.013636,65,99.0
+5,one another;,0.44,0.542727,0.100392,0.012727,66,99.0
+5,of the operation,0.439608,0.560606,0.126667,0.011515,67,99.0
+5,information:,0.437255,0.575758,0.099216,0.010303,68,99.0
+5,"to both, that",0.432157,0.590303,0.101176,0.014545,69,98.0
+5,"powerful financial,",0.420784,0.606667,0.154902,0.014545,70,99.0
+5,priorities relevant,0.404706,0.63697,0.16,0.017879,71,97.0
+5,s support to,0.446667,0.655152,0.089804,0.014242,72,99.0
+5,partnership;,0.40902,0.671515,0.095686,0.015152,73,99.0
+5,"mind, the Mayor",0.423922,0.700303,0.132157,0.016364,74,98.0
+5,these two,0.440784,0.716667,0.08,0.015455,75,98.0
+5,of Understanding.,0.438824,0.731818,0.144314,0.017273,76,99.0
+5,will go into,0.43451,0.777576,0.094118,0.018485,77,98.0
+5,Signed in,0.456078,0.794848,0.078824,0.014545,78,97.0
+5,OF New,0.536078,0.259697,0.089804,0.013333,79,85.0
+5,OF THE MayOr,0.527059,0.276061,0.144706,0.011515,80,73.0
+5,N.Y. 1OOO7,0.565882,0.291212,0.091765,0.012121,81,89.0
+5,SISTER,0.573333,0.353939,0.076078,0.014545,82,98.0
+5,of Understanding,0.542745,0.369394,0.152941,0.014545,83,99.0
+5,York City and,0.534902,0.400606,0.114118,0.013333,84,99.0
+5,for these two,0.545882,0.417273,0.106275,0.011818,85,99.0
+5,between the,0.537647,0.43303,0.102353,0.010303,86,96.0
+5,"partnerships, as it",0.491765,0.448182,0.138039,0.012121,87,98.0
+5,exchanges between,0.496863,0.478788,0.166667,0.014848,88,99.0
+5,cross-section,0.546275,0.497273,0.105098,0.009091,89,99.0
+5,of new social.,0.536863,0.512727,0.11098,0.011515,90,97.0
+5,cities citizens,0.552157,0.528788,0.112157,0.009091,91,99.0
+5,of the cities',0.56549,0.559697,0.103529,0.012727,92,97.0
+5,can generate,0.533333,0.590303,0.105882,0.014545,93,98.0
+5,social and,0.575686,0.606667,0.085098,0.014545,94,99.0
+5,to both,0.564706,0.63697,0.054118,0.017879,95,97.0
+5,community-led,0.536471,0.655758,0.119608,0.013636,96,99.0
+5,of the City,0.556078,0.700303,0.09098,0.016364,97,98.0
+5,cities are u united,0.520784,0.716667,0.129804,0.015455,98,98.0
+5,effect from the,0.528627,0.777576,0.121569,0.018485,99,98.0
+5,March of 2001,0.534902,0.794848,0.116471,0.014545,100,97.0
+5,York,0.625882,0.259697,0.056078,0.013333,101,85.0
+5,CITY,0.649412,0.353939,0.058039,0.014545,102,98.0
+5,London will,0.64902,0.400606,0.100784,0.013333,103,99.0
+5,great,0.652157,0.417273,0.04902,0.011818,104,99.0
+5,two will be one,0.64,0.43303,0.120392,0.010303,105,96.0
+5,strives to:,0.629804,0.448182,0.079608,0.012121,106,98.0
+5,London,0.663529,0.478788,0.088235,0.014848,107,99.0
+5,of the citizens,0.654118,0.495152,0.113725,0.012121,108,99.0
+5,"economic,",0.652941,0.511818,0.085098,0.012727,109,99.0
+5,to share,0.66902,0.527273,0.069412,0.013636,110,99.0
+5,various,0.66902,0.559697,0.066667,0.012727,111,97.0
+5,new initiatives,0.639216,0.590303,0.118431,0.014545,112,98.0
+5,cultural,0.660784,0.606667,0.066275,0.014545,113,99.0
+5,London and,0.618824,0.63697,0.12902,0.017879,114,97.0
+5,programs,0.656078,0.655758,0.083922,0.013636,115,99.0
+5,of New York,0.647059,0.700303,0.107843,0.016364,116,98.0
+5,by an official,0.650588,0.717273,0.111373,0.013636,117,99.0
+5,date of,0.650196,0.777576,0.062353,0.018485,118,98.0
+5,Ken,0.668627,0.873636,0.037647,0.018182,119,98.0
+5,Mayor,0.703922,0.890606,0.06,0.01697,120,99.0
+5,London,0.698039,0.906364,0.069412,0.016061,121,99.0
+5,PARTNERSHIP,0.707451,0.353939,0.143137,0.014545,122,98.0
+5,foster mutually,0.749804,0.400606,0.125098,0.013333,123,99.0
+5,cosmopolitan entities,0.701176,0.417273,0.166275,0.011818,124,99.0
+5,of the most,0.760392,0.43303,0.093333,0.010303,125,96.0
+5,and New York,0.751765,0.478788,0.095686,0.014848,126,99.0
+5,of both;,0.767843,0.495152,0.06549,0.012121,127,98.0
+5,academic and,0.738039,0.511818,0.112157,0.012727,128,99.0
+5,their experiences,0.738431,0.527273,0.139608,0.013636,129,99.0
+5,government,0.735686,0.559697,0.1,0.012727,130,97.0
+5,to further,0.757647,0.590303,0.08549,0.014545,131,98.0
+5,relationships,0.727059,0.606667,0.10902,0.014545,132,99.0
+5,New York City;,0.747843,0.63697,0.090196,0.017879,133,97.0
+5,that advance,0.74,0.655758,0.106667,0.013636,134,99.0
+5,and the,0.754902,0.700303,0.065098,0.016364,135,98.0
+5,partnership,0.761961,0.717273,0.1,0.013636,136,99.0
+5,signatures.,0.712549,0.777576,0.086275,0.018485,137,98.0
+5,G,0.701569,0.82697,0.100784,0.038485,138,55.0
+5,Livingstone,0.706275,0.873636,0.1,0.018182,139,98.0
+5,Toolkit,0.82902,0.069091,0.133333,0.032424,140,99.0
+5,City so,0.847451,0.478788,0.055294,0.014848,141,99.0
+5,as a,0.878039,0.527273,0.031765,0.013636,142,99.0
+5,agencies,0.835686,0.559697,0.07098,0.012727,143,97.0
+5,and,0.843137,0.590303,0.028627,0.014545,144,98.0
+5,between,0.836078,0.606667,0.06549,0.014545,145,99.0
+5,the,0.846667,0.655758,0.024706,0.013636,146,99.0
+5,Mayor of,0.82,0.700303,0.076863,0.016364,147,98.0
+5,by the,0.861961,0.717273,0.048235,0.013636,148,99.0
+6,SisterCities Partnership Agreement,0.167843,0.025758,0.793725,0.03697,1,99.0
+6,?,0.072157,0.689091,0.383529,0.282121,2,33.0
+6,IN TE RN A T,0.17098,0.066667,0.146667,0.009091,3,88.0
+6,Connect globally.,0.167451,0.084848,0.13451,0.016364,4,99.0
+6,"California, In accordance USA,",0.216471,0.482121,0.138824,0.021515,5,98.0
+6,"purposes: that a ""Sister City",0.216078,0.502727,0.121176,0.024545,6,97.0
+6,the people (1) of Long to,0.218431,0.531515,0.114118,0.024242,7,97.0
+6,relations between (2) to,0.217255,0.561212,0.107843,0.025152,8,98.0
+6,"information professional, in",0.217647,0.583636,0.10549,0.023333,9,96.0
+6,efucational (3) activities to,0.218431,0.612424,0.129412,0.023939,10,98.0
+6,this document This Sister,0.217647,0.642424,0.11451,0.025152,11,97.0
+6,Mayor of San,0.218039,0.663333,0.089412,0.015758,12,96.0
+6,IO N A L,0.317647,0.066667,0.09098,0.009091,13,88.0
+6,Thrive locally.,0.301961,0.084848,0.108627,0.016364,14,99.0
+6,Pister,0.321176,0.308182,0.096863,0.022424,15,96.0
+6,City,0.378431,0.374545,0.058431,0.020303,16,99.0
+6,City of,0.321569,0.425758,0.102353,0.019091,17,98.0
+6,"Ecuador,",0.345882,0.447273,0.126667,0.014848,18,98.0
+6,and the with City the,0.322745,0.482121,0.100784,0.021515,19,98.0
+6,"Agreement""",0.337255,0.502727,0.081961,0.012121,20,97.0
+6,promote Beach and and the,0.320784,0.531515,0.098824,0.024242,21,97.0
+6,promote the fwo cities,0.324314,0.561212,0.083922,0.025152,22,97.0
+6,"technical, a wide youth, sariety and",0.300784,0.583636,0.124314,0.023333,23,96.0
+6,foster between and the,0.321176,0.612424,0.102745,0.023939,24,98.0
+6,City has been Agreement duly,0.331373,0.642424,0.1,0.025152,25,97.0
+6,"Pablo de Manta,",0.307451,0.663333,0.099608,0.015758,26,96.0
+6,Cit,0.39098,0.201515,0.098039,0.055455,27,89.0
+6,Lity,0.432157,0.30697,0.072549,0.033333,28,96.0
+6,between,0.464706,0.35,0.062745,0.013636,29,98.0
+6,of Long,0.436863,0.374545,0.114902,0.020303,30,99.0
+6,"California,",0.400392,0.396667,0.155686,0.016061,31,99.0
+6,San and,0.423922,0.412727,0.087451,0.032121,32,98.0
+6,South,0.472549,0.447273,0.083529,0.014848,33,98.0
+6,authorization of San Pablo and de,0.403922,0.482121,0.110196,0.021515,34,98.0
+6,between the two,0.419216,0.502727,0.102745,0.012121,35,97.0
+6,expand people the of effective San,0.402353,0.531515,0.127059,0.024242,36,97.0
+6,"international and their respective goodwill,",0.38549,0.561212,0.152549,0.025152,37,97.0
+6,"of other economic, endeavors;and",0.423922,0.583636,0.127451,0.023333,38,96.0
+6,"encourage two cities; charitable,",0.391373,0.612424,0.138824,0.023939,39,98.0
+6,executed shall be by officially the,0.401176,0.642424,0.135294,0.025152,40,97.0
+6,"Ecuador, South",0.407059,0.663333,0.098431,0.015758,41,96.0
+6,of Long,0.50902,0.210303,0.105098,0.057576,42,89.0
+6,Agreement,0.514902,0.308182,0.188627,0.032121,43,96.0
+6,the,0.527451,0.35,0.022353,0.013636,44,98.0
+6,Beach,0.551765,0.374545,0.082745,0.020303,45,99.0
+6,USA,0.556078,0.396667,0.056471,0.016061,46,99.0
+6,Pablo the de,0.484314,0.412727,0.126275,0.032121,47,98.0
+6,America,0.556078,0.447273,0.10902,0.014848,48,98.0
+6,"Manta, approval Ecuador, expressed",0.507451,0.482121,0.128235,0.021515,49,98.0
+6,cities is hereby,0.521961,0.502727,0.098431,0.012121,50,97.0
+6,Pablo and de mutually Manta; and,0.504706,0.531515,0.12,0.024242,51,97.0
+6,"natioms understanding, by the",0.532549,0.561212,0.108235,0.025152,52,97.0
+6,"social, cultural,",0.518039,0.583636,0.109412,0.012121,53,96.0
+6,"scientific, trade",0.530196,0.612424,0.121961,0.013939,54,98.0
+6,Mayor established of Long and,0.507059,0.642424,0.125098,0.025152,55,97.0
+6,America.,0.50549,0.663333,0.053333,0.015758,56,96.0
+6,Beperly,0.587059,0.735758,0.063137,0.015758,57,99.0
+6,"Mayor, City",0.542745,0.750909,0.096078,0.01303,58,98.0
+6,"Mayor, Ing. Jorge",0.504314,0.824545,0.098431,0.028485,59,97.0
+6,"Ecuador, City of",0.552157,0.840303,0.070588,0.025152,60,99.0
+6,Dated:,0.545882,0.883333,0.051765,0.011515,61,99.0
+6,alifornia,0.561569,0.257879,0.127451,0.028485,62,99.0
+6,Manta,0.610588,0.425758,0.081961,0.019091,63,98.0
+6,"South bry America, the City",0.615686,0.482121,0.103137,0.021515,64,97.0
+6,established for,0.620392,0.502727,0.1,0.012121,65,97.0
+6,beneficial cooperation,0.612549,0.531515,0.137647,0.012727,66,98.0
+6,exchange and expanded of,0.620784,0.561212,0.119216,0.025152,67,98.0
+6,"municipal,",0.627451,0.583636,0.08,0.012121,68,96.0
+6,"and commerce,",0.652157,0.612424,0.075686,0.013939,69,98.0
+6,"Beach, shall California, become",0.606667,0.642424,0.138039,0.025152,70,97.0
+6,Hiee,0.553333,0.697879,0.179216,0.037273,71,54.0
+6,bNeill,0.647059,0.735758,0.064706,0.011818,72,90.0
+6,of Long USA,0.643137,0.750909,0.066275,0.025455,73,98.0
+6,"California,",0.583137,0.766364,0.088627,0.011515,74,99.0
+6,Dulus,0.596471,0.792424,0.080784,0.027273,75,69.0
+6,O.Zambrano Pablo,0.603922,0.824545,0.107843,0.025152,76,98.0
+6,San South America,0.623922,0.839394,0.116078,0.025455,77,99.0
+6,September,0.600392,0.881818,0.107451,0.013939,78,97.0
+6,Beach,0.715686,0.210303,0.148235,0.048182,79,89.0
+6,"of it Long is declared Beach,",0.708627,0.482121,0.091373,0.021515,80,98.0
+6,the following,0.720392,0.502727,0.079608,0.012121,81,97.0
+6,between,0.750196,0.531515,0.050196,0.012727,82,98.0
+6,"people, business ideas, and",0.698824,0.561212,0.102353,0.025152,83,97.0
+6,"environmental,",0.707451,0.583636,0.092941,0.012121,84,96.0
+6,literary and,0.727843,0.612424,0.073333,0.013939,85,98.0
+6,"effective USA, and when the",0.712157,0.642424,0.08902,0.025152,86,97.0
+6,Beach,0.704314,0.750909,0.049804,0.012727,87,97.0
+6,Cedeño,0.707451,0.824242,0.063137,0.011212,88,96.0
+6,de Manta,0.706667,0.839091,0.076471,0.009394,89,99.0
+6,"19,2000",0.707843,0.881818,0.04,0.013939,90,97.0
+6,Toolkit,0.82902,0.069091,0.133333,0.032424,91,99.0
+7,SisterCities Partnership Agreement,0.167843,0.025758,0.793725,0.03697,1,99.0
+7,IN TE RN A,0.17098,0.066667,0.125882,0.009091,2,90.0
+7,Connect globally.,0.167843,0.085152,0.134118,0.015455,3,98.0
+7,adopted by,0.198431,0.211212,0.085882,0.017576,4,99.0
+7,and,0.195686,0.256364,0.03451,0.019091,5,99.0
+7,On this,0.252549,0.358182,0.06,0.021515,6,98.0
+7,the traditional,0.196471,0.375758,0.112549,0.019394,7,98.0
+7,aspiration to,0.198824,0.393939,0.105098,0.014848,8,98.0
+7,Richard M.,0.198431,0.409697,0.089804,0.015455,9,97.0
+7,of the City of,0.198824,0.426364,0.105098,0.014545,10,97.0
+7,sister cities,0.198431,0.441818,0.111765,0.015152,11,98.0
+7,The,0.25451,0.471212,0.034902,0.017576,12,98.0
+7,equality and,0.198824,0.487273,0.098431,0.017576,13,98.0
+7,broaden economic,0.199608,0.504848,0.139608,0.014848,14,98.0
+7,The two,0.253725,0.534545,0.062353,0.019697,15,98.0
+7,and trade,0.198824,0.553333,0.077647,0.014848,16,98.0
+7,In,0.253333,0.582121,0.023922,0.017576,17,99.0
+7,dance and other,0.199216,0.600606,0.125098,0.011818,18,96.0
+7,In,0.253333,0.628182,0.024314,0.017576,19,98.0
+7,within educational,0.199216,0.647273,0.137647,0.01303,20,98.0
+7,In,0.252549,0.675152,0.023529,0.018182,21,99.0
+7,"technology,",0.198039,0.69303,0.092941,0.015152,22,97.0
+7,further,0.196863,0.711818,0.061569,0.012424,23,99.0
+7,THE HONORABLE,0.182353,0.818485,0.154902,0.016061,24,97.0
+7,MAYOR OF,0.196078,0.834242,0.088235,0.013939,25,97.0
+7,T IO N A L,0.296863,0.066667,0.111765,0.009091,26,90.0
+7,Thrive locally.,0.301961,0.085152,0.108235,0.015455,27,98.0
+7,the tenth,0.312549,0.358182,0.069412,0.021515,28,98.0
+7,links of,0.30902,0.375758,0.063922,0.019394,29,98.0
+7,wark in unison,0.303922,0.393939,0.119608,0.014848,30,98.0
+7,"Daley, Mayor",0.288235,0.409697,0.110588,0.015455,31,97.0
+7,"Shenyang, on",0.303922,0.426364,0.102745,0.014545,32,97.0
+7,agreement,0.310196,0.441818,0.06549,0.015152,33,98.0
+7,City of Chicago,0.289412,0.471212,0.127843,0.017576,34,98.0
+7,mutual benefit,0.297255,0.487273,0.116863,0.017576,35,98.0
+7,cooperation,0.339216,0.504848,0.093725,0.014848,36,98.0
+7,cities do,0.316078,0.534545,0.066667,0.019697,37,98.0
+7,relations between,0.276471,0.553333,0.166275,0.014848,38,98.0
+7,"addition, exchanges",0.277255,0.582121,0.156078,0.017576,39,99.0
+7,cultural,0.324314,0.600606,0.061569,0.011818,40,96.0
+7,"addition, exchanges",0.277647,0.628182,0.156078,0.017576,41,98.0
+7,institutions,0.336863,0.647273,0.087059,0.01303,42,98.0
+7,"addition, we declare",0.276078,0.675152,0.153725,0.018182,43,99.0
+7,"sports, health,",0.29098,0.69303,0.116471,0.015152,44,97.0
+7,development of,0.258431,0.711818,0.115686,0.012424,45,99.0
+7,RICHARD,0.337255,0.818485,0.082745,0.016061,46,97.0
+7,CHICAGO,0.284314,0.834242,0.076078,0.013939,47,97.0
+7,REAFFIRMATION OF,0.323922,0.162727,0.199216,0.018485,48,97.0
+7,THE HONORABLE,0.394902,0.212727,0.160392,0.014545,49,99.0
+7,MAYOR,0.472157,0.23,0.070588,0.013333,50,98.0
+7,THE HONORABLE,0.401569,0.258788,0.163922,0.014242,51,99.0
+7,MAYOR,0.463529,0.272424,0.07098,0.01303,52,99.0
+7,anniversary of,0.381961,0.358182,0.118431,0.021515,53,98.0
+7,friendship between,0.372941,0.375758,0.156078,0.019394,54,98.0
+7,for the,0.423529,0.393939,0.063529,0.014848,55,98.0
+7,of the City of,0.398824,0.409697,0.11451,0.015455,56,97.0
+7,this fifth day,0.406667,0.426364,0.103137,0.014545,57,97.0
+7,between the City,0.375686,0.441818,0.138824,0.015152,58,98.0
+7,and the City,0.417255,0.471212,0.107059,0.017576,59,98.0
+7,will continue,0.414118,0.487273,0.105098,0.017576,60,98.0
+7,and cultural,0.432941,0.504848,0.094118,0.014848,61,98.0
+7,hereby declare,0.382745,0.534545,0.121569,0.019697,62,98.0
+7,Chicago and,0.442745,0.553333,0.066667,0.014848,63,98.0
+7,will be,0.433333,0.582121,0.052549,0.017576,64,99.0
+7,activities.,0.385882,0.600606,0.068235,0.011818,65,96.0
+7,will be,0.433725,0.628182,0.052549,0.017576,66,98.0
+7,encouraged.,0.423922,0.647273,0.09451,0.01303,67,98.0
+7,our intention,0.429804,0.675152,0.102745,0.018182,68,99.0
+7,youth and any,0.407451,0.69303,0.116471,0.015152,69,97.0
+7,friendship between,0.374118,0.711818,0.144314,0.012424,70,99.0
+7,M. DALEY,0.42,0.818485,0.080392,0.016061,71,97.0
+7,SISTER,0.523137,0.162727,0.07451,0.018485,72,97.0
+7,RICHARD,0.555294,0.212727,0.092549,0.014545,73,99.0
+7,OF CHICAGO,0.542745,0.23,0.115294,0.013333,74,98.0
+7,ZHANG,0.56549,0.258788,0.069412,0.014242,75,99.0
+7,OF SHENYANG,0.53451,0.272424,0.131765,0.01303,76,99.0
+7,ON,0.549412,0.295455,0.032157,0.016667,77,99.0
+7,"JUNE 5, 1995",0.500784,0.321818,0.128235,0.015758,78,96.0
+7,the signing of a,0.500392,0.358182,0.120784,0.021515,79,98.0
+7,Chicago and,0.52902,0.375758,0.098824,0.019394,80,98.0
+7,benefit of their,0.487059,0.393939,0.12902,0.014848,81,98.0
+7,"Chicago, and",0.513333,0.409697,0.106667,0.015455,82,97.0
+7,"of June I995,",0.509804,0.426364,0.11098,0.014545,83,97.0
+7,of Chicago and,0.51451,0.441818,0.103529,0.015152,84,98.0
+7,of Shenyang,0.524314,0.471212,0.107059,0.017576,85,98.0
+7,to develop a,0.519216,0.487273,0.092941,0.017576,86,98.0
+7,erchanges,0.527059,0.504848,0.083137,0.014848,87,98.0
+7,their interest in,0.504314,0.534545,0.121961,0.019697,88,98.0
+7,Shenyang.,0.509412,0.553333,0.076078,0.014848,89,98.0
+7,promoted in the,0.485882,0.582121,0.127059,0.017576,90,99.0
+7,promoted in,0.486275,0.628182,0.098824,0.017576,91,98.0
+7,to promote,0.532549,0.675152,0.086275,0.018182,92,99.0
+7,areas that,0.523922,0.69303,0.086275,0.015152,93,97.0
+7,the people of,0.518431,0.711818,0.1,0.012424,94,99.0
+7,THE,0.587843,0.818485,0.039216,0.013939,95,99.0
+7,MAYOR,0.588235,0.834545,0.061961,0.012424,96,97.0
+7,CITIES,0.597647,0.162727,0.119216,0.018485,97,97.0
+7,M. DALEY,0.647843,0.212727,0.085882,0.014545,98,99.0
+7,RONGMAO,0.634902,0.258788,0.09098,0.014242,99,99.0
+7,sister city,0.621176,0.358182,0.078431,0.021515,100,98.0
+7,Shenyang,0.627843,0.375758,0.080784,0.019394,101,98.0
+7,cities and,0.616078,0.393939,0.080392,0.014848,102,98.0
+7,the Honorable,0.62,0.409697,0.119608,0.015455,103,97.0
+7,do hereby,0.620784,0.426364,0.077647,0.014545,104,97.0
+7,the City of,0.618039,0.441818,0.082353,0.015152,105,98.0
+7,on the basis,0.631373,0.471212,0.103137,0.017576,106,98.0
+7,sister cities,0.612157,0.487273,0.098824,0.017576,107,98.0
+7,between the two,0.610196,0.504848,0.122353,0.014848,108,98.0
+7,exploring the,0.626275,0.534545,0.103137,0.019697,109,98.0
+7,area of the,0.612941,0.582121,0.090588,0.017576,110,99.0
+7,education and the,0.585098,0.628182,0.142353,0.017576,111,98.0
+7,exchanges in,0.618824,0.675152,0.102745,0.018182,112,99.0
+7,will contribute,0.610196,0.69303,0.114118,0.015152,113,97.0
+7,our two cities.,0.618431,0.711818,0.1,0.012424,114,99.0
+7,HONORABLE,0.627059,0.818485,0.107059,0.013939,115,99.0
+7,OF,0.650196,0.834545,0.027059,0.012424,116,97.0
+7,DECLARATION,0.716863,0.162727,0.092941,0.018485,117,97.0
+7,"agreement, in",0.699608,0.358182,0.104706,0.021515,118,98.0
+7,and to,0.708627,0.375758,0.052549,0.019394,119,98.0
+7,"nations, the",0.696471,0.393939,0.094902,0.014848,120,98.0
+7,Zhang,0.739608,0.409697,0.055294,0.015455,121,97.0
+7,acknowledge,0.698431,0.426364,0.1,0.014545,122,97.0
+7,Shenyang,0.700392,0.441818,0.072549,0.015152,123,98.0
+7,of friendly,0.73451,0.471212,0.08902,0.017576,124,98.0
+7,relationship,0.71098,0.487273,0.097647,0.017576,125,98.0
+7,cities.,0.732549,0.504848,0.041569,0.014848,126,98.0
+7,establishment,0.729412,0.534545,0.106667,0.019697,127,98.0
+7,arts such as,0.703529,0.582121,0.103922,0.017576,128,99.0
+7,establishment,0.727451,0.628182,0.107843,0.017576,129,98.0
+7,such fields,0.721569,0.675152,0.089412,0.018182,130,99.0
+7,to the,0.724314,0.69303,0.05098,0.015152,131,97.0
+7,ZHANG,0.734118,0.818485,0.061961,0.013939,132,99.0
+7,SHENYANG,0.677255,0.834545,0.088235,0.012424,133,97.0
+7,Toolkit,0.82902,0.069394,0.132941,0.031515,134,99.0
+7,order to further,0.804314,0.358182,0.127059,0.021515,135,98.0
+7,reaffirm their mutual,0.761176,0.375758,0.16902,0.019394,136,98.0
+7,Honorable Mayor,0.791373,0.393939,0.136863,0.014848,137,98.0
+7,"Rongmao, Mayor",0.794902,0.409697,0.133333,0.015455,138,97.0
+7,and reaffir.n the,0.798431,0.426364,0.128235,0.014545,139,97.0
+7,"cooperation,",0.823529,0.471212,0.1,0.017576,140,98.0
+7,to promote and,0.808627,0.487273,0.118431,0.017576,141,98.0
+7,of business,0.836078,0.534545,0.089412,0.019697,142,98.0
+7,"exhibits, music,",0.807451,0.582121,0.113725,0.017576,143,99.0
+7,of contacts,0.835294,0.628182,0.088235,0.017576,144,98.0
+7,as science and,0.81098,0.675152,0.113333,0.018182,145,99.0
+7,prosperity and the,0.775294,0.69303,0.147059,0.015152,146,97.0
+7,RONGMAO,0.796078,0.818485,0.081569,0.013939,147,99.0