Sync: Revised OCR line parsing for single column pages with local OCR. Minor changes to docker compose

.coveragerc

@@ -0,0 +1,56 @@
+[run]
+source = .
+omit = 
+    */tests/*
+    */test/*
+    */__pycache__/*
+    */venv/*
+    */env/*
+    */build/*
+    */dist/*
+    */cdk/*
+    */docs/*
+    */example_data/*
+    */examples/*
+    */feedback/*
+    */logs/*
+    */old_code/*
+    */output/*
+    */tmp/*
+    */usage/*
+    */tld/*
+    */tesseract/*
+    */poppler/*
+    config*.py
+    setup.py
+    lambda_entrypoint.py
+    entrypoint.sh
+    cli_redact.py
+    load_dynamo_logs.py
+    load_s3_logs.py
+    *.spec
+    Dockerfile
+    *.qmd
+    *.md
+    *.txt
+    *.yml
+    *.yaml
+    *.json
+    *.csv
+    *.env
+    *.bat
+    *.ps1
+    *.sh
+
+[report]
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    if self.debug:
+    if settings.DEBUG
+    raise AssertionError
+    raise NotImplementedError
+    if 0:
+    if __name__ == .__main__.:
+    class .*\bProtocol\):
+    @(abc\.)?abstractmethod

.dockerignore

@@ -0,0 +1,50 @@
+*.url
+*.ipynb
+*.pyc
+*.qmd
+_quarto.yml
+quarto_site/*
+src/*
+redaction_deps/*
+.venv/*
+examples/*
+processing/*
+tools/__pycache__/*
+old_code/*
+tesseract/*
+poppler/*
+build/*
+dist/*
+docs/*
+build_deps/*
+user_guide/*
+_extensions/*
+doc_redaction.egg-info/*
+.venv_pypi_test/*
+cdk/config/*
+tld/*
+cdk/config/*
+cdk/cdk.out/*
+cdk/archive/*
+cdk.json
+cdk.context.json
+.quarto/*
+logs/
+output/
+input/
+feedback/
+config/
+usage/
+test/config/*
+test/feedback/*
+test/input/*
+test/logs/*
+test/output/*
+test/tmp/*
+test/usage/*
+.ruff_cache/*
+model_cache/*
+sanitized_file/*
+src/doc_redaction.egg-info/*
+docker_compose/*
+skills/example_prompts/*

.gitattributes

@@ -0,0 +1,8 @@
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.xls filter=lfs diff=lfs merge=lfs -text
+*.xlsx filter=lfs diff=lfs merge=lfs -text
+*.docx filter=lfs diff=lfs merge=lfs -text
+*.doc filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.ico filter=lfs diff=lfs merge=lfs -text

.github/scripts/setup_test_data.py

@@ -0,0 +1,320 @@
+#!/usr/bin/env python3
+"""
+Setup script for GitHub Actions test data.
+Creates dummy test files when example data is not available.
+"""
+
+import os
+import sys
+
+import pandas as pd
+
+
+def create_directories():
+    """Create necessary directories."""
+    dirs = ["doc_redaction/example_data", "doc_redaction/example_data/example_outputs"]
+
+    for dir_path in dirs:
+        os.makedirs(dir_path, exist_ok=True)
+        print(f"Created directory: {dir_path}")
+
+
+def create_dummy_pdf():
+    """Create dummy PDFs for testing."""
+
+    # Install reportlab if not available
+    try:
+        from reportlab.lib.pagesizes import letter
+        from reportlab.pdfgen import canvas
+    except ImportError:
+        import subprocess
+
+        subprocess.check_call(["pip", "install", "reportlab"])
+        from reportlab.lib.pagesizes import letter
+        from reportlab.pdfgen import canvas
+
+    try:
+        # Create the main test PDF
+        pdf_path = "doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf"
+        print(f"Creating PDF: {pdf_path}")
+        print(f"Directory exists: {os.path.exists('doc_redaction/example_data')}")
+
+        c = canvas.Canvas(pdf_path, pagesize=letter)
+        c.drawString(100, 750, "This is a test document for redaction testing.")
+        c.drawString(100, 700, "Email: test@example.com")
+        c.drawString(100, 650, "Phone: 123-456-7890")
+        c.drawString(100, 600, "Name: John Doe")
+        c.drawString(100, 550, "Address: 123 Test Street, Test City, TC 12345")
+        c.showPage()
+
+        # Add second page
+        c.drawString(100, 750, "Second page content")
+        c.drawString(100, 700, "More test data: jane.doe@example.com")
+        c.drawString(100, 650, "Another phone: 987-654-3210")
+        c.save()
+
+        print(f"Created dummy PDF: {pdf_path}")
+
+        # Create Partnership Agreement Toolkit PDF
+        partnership_pdf_path = (
+            "doc_redaction/example_data/Partnership-Agreement-Toolkit_0_0.pdf"
+        )
+        print(f"Creating PDF: {partnership_pdf_path}")
+        c = canvas.Canvas(partnership_pdf_path, pagesize=letter)
+        c.drawString(100, 750, "Partnership Agreement Toolkit")
+        c.drawString(100, 700, "This is a test partnership agreement document.")
+        c.drawString(100, 650, "Contact: partnership@example.com")
+        c.drawString(100, 600, "Phone: (555) 123-4567")
+        c.drawString(100, 550, "Address: 123 Partnership Street, City, State 12345")
+        c.showPage()
+
+        # Add second page
+        c.drawString(100, 750, "Page 2 - Partnership Details")
+        c.drawString(100, 700, "More partnership information here.")
+        c.drawString(100, 650, "Contact: info@partnership.org")
+        c.showPage()
+
+        # Add third page
+        c.drawString(100, 750, "Page 3 - Terms and Conditions")
+        c.drawString(100, 700, "Terms and conditions content.")
+        c.drawString(100, 650, "Legal contact: legal@partnership.org")
+        c.save()
+
+        print(f"Created dummy PDF: {partnership_pdf_path}")
+
+        # Create Graduate Job Cover Letter PDF
+        cover_letter_pdf_path = (
+            "doc_redaction/example_data/graduate-job-example-cover-letter.pdf"
+        )
+        print(f"Creating PDF: {cover_letter_pdf_path}")
+        c = canvas.Canvas(cover_letter_pdf_path, pagesize=letter)
+        c.drawString(100, 750, "Cover Letter Example")
+        c.drawString(100, 700, "Dear Hiring Manager,")
+        c.drawString(100, 650, "I am writing to apply for the position.")
+        c.drawString(100, 600, "Contact: applicant@example.com")
+        c.drawString(100, 550, "Phone: (555) 987-6543")
+        c.drawString(100, 500, "Address: 456 Job Street, Employment City, EC 54321")
+        c.drawString(100, 450, "Sincerely,")
+        c.drawString(100, 400, "John Applicant")
+        c.save()
+
+        print(f"Created dummy PDF: {cover_letter_pdf_path}")
+
+    except ImportError:
+        print("ReportLab not available, skipping PDF creation")
+        # Create simple text files instead
+        with open(
+            "doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf",
+            "w",
+        ) as f:
+            f.write("This is a dummy PDF file for testing")
+
+        with open(
+            "doc_redaction/example_data/Partnership-Agreement-Toolkit_0_0.pdf",
+            "w",
+        ) as f:
+            f.write("This is a dummy Partnership Agreement PDF file for testing")
+
+        with open(
+            "doc_redaction/example_data/graduate-job-example-cover-letter.pdf",
+            "w",
+        ) as f:
+            f.write("This is a dummy cover letter PDF file for testing")
+
+        print("Created dummy text files instead of PDFs")
+
+
+def create_dummy_csv():
+    """Create dummy CSV files for testing."""
+    # Main CSV
+    csv_data = {
+        "Case Note": [
+            "Client visited for consultation regarding housing issues",
+            "Follow-up appointment scheduled for next week",
+            "Documentation submitted for review",
+        ],
+        "Client": ["John Smith", "Jane Doe", "Bob Johnson"],
+        "Date": ["2024-01-15", "2024-01-16", "2024-01-17"],
+    }
+    df = pd.DataFrame(csv_data)
+    df.to_csv("doc_redaction/example_data/combined_case_notes.csv", index=False)
+    print("Created dummy CSV: doc_redaction/example_data/combined_case_notes.csv")
+
+    # Lambeth CSV
+    lambeth_data = {
+        "text": [
+            "Lambeth 2030 vision document content",
+            "Our Future Our Lambeth strategic plan",
+            "Community engagement and development",
+        ],
+        "page": [1, 2, 3],
+    }
+    df_lambeth = pd.DataFrame(lambeth_data)
+    df_lambeth.to_csv(
+        "doc_redaction/example_data/Lambeth_2030-Our_Future_Our_Lambeth.pdf.csv",
+        index=False,
+    )
+    print(
+        "Created dummy CSV: doc_redaction/example_data/Lambeth_2030-Our_Future_Our_Lambeth.pdf.csv"
+    )
+
+
+def create_dummy_word_doc():
+    """Create dummy Word document."""
+    try:
+        from docx import Document
+
+        doc = Document()
+        doc.add_heading("Test Document for Redaction", 0)
+        doc.add_paragraph("This is a test document for redaction testing.")
+        doc.add_paragraph("Contact Information:")
+        doc.add_paragraph("Email: test@example.com")
+        doc.add_paragraph("Phone: 123-456-7890")
+        doc.add_paragraph("Name: John Doe")
+        doc.add_paragraph("Address: 123 Test Street, Test City, TC 12345")
+
+        doc.save(
+            "doc_redaction/example_data/Bold minimalist professional cover letter.docx"
+        )
+        print("Created dummy Word document")
+
+    except ImportError:
+        print("python-docx not available, skipping Word document creation")
+
+
+def create_allow_deny_lists():
+    """Create dummy allow/deny lists."""
+    # Allow lists
+    allow_data = {"word": ["test", "example", "document"]}
+    pd.DataFrame(allow_data).to_csv(
+        "doc_redaction/example_data/test_allow_list_graduate.csv", index=False
+    )
+    pd.DataFrame(allow_data).to_csv(
+        "doc_redaction/example_data/test_allow_list_partnership.csv", index=False
+    )
+    print("Created allow lists")
+
+    # Deny lists
+    deny_data = {"word": ["sensitive", "confidential", "private"]}
+    pd.DataFrame(deny_data).to_csv(
+        "doc_redaction/example_data/partnership_toolkit_redact_custom_deny_list.csv",
+        index=False,
+    )
+    pd.DataFrame(deny_data).to_csv(
+        "doc_redaction/example_data/Partnership-Agreement-Toolkit_test_deny_list_para_single_spell.csv",
+        index=False,
+    )
+    print("Created deny lists")
+
+    # Whole page redaction list
+    page_data = {"page": [1, 2]}
+    pd.DataFrame(page_data).to_csv(
+        "doc_redaction/example_data/partnership_toolkit_redact_some_pages.csv",
+        index=False,
+    )
+    print("Created whole page redaction list")
+
+
+def create_ocr_output():
+    """Create dummy OCR output CSV."""
+    ocr_data = {
+        "page": [1, 2, 3],
+        "text": [
+            "This is page 1 content with some text",
+            "This is page 2 content with different text",
+            "This is page 3 content with more text",
+        ],
+        "left": [0.1, 0.3, 0.5],
+        "top": [0.95, 0.92, 0.88],
+        "width": [0.05, 0.02, 0.02],
+        "height": [0.01, 0.02, 0.02],
+        "line": [1, 2, 3],
+    }
+    df = pd.DataFrame(ocr_data)
+    df.to_csv(
+        "doc_redaction/example_data/example_outputs/doubled_output_joined.pdf_ocr_output.csv",
+        index=False,
+    )
+    print("Created dummy OCR output CSV")
+
+
+def create_dummy_image():
+    """Create dummy image for testing."""
+    try:
+        from PIL import Image, ImageDraw, ImageFont
+
+        img = Image.new("RGB", (800, 600), color="white")
+        draw = ImageDraw.Draw(img)
+
+        # Try to use a system font
+        try:
+            font = ImageFont.truetype(
+                "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 20
+            )
+        except Exception as e:
+            print(f"Error loading DejaVuSans font: {e}")
+            try:
+                font = ImageFont.truetype("/System/Library/Fonts/Arial.ttf", 20)
+            except Exception as e:
+                print(f"Error loading Arial font: {e}")
+                font = ImageFont.load_default()
+
+        # Add text to image
+        draw.text((50, 50), "Test Document for Redaction", fill="black", font=font)
+        draw.text((50, 100), "Email: test@example.com", fill="black", font=font)
+        draw.text((50, 150), "Phone: 123-456-7890", fill="black", font=font)
+        draw.text((50, 200), "Name: John Doe", fill="black", font=font)
+        draw.text((50, 250), "Address: 123 Test Street", fill="black", font=font)
+
+        img.save("doc_redaction/example_data/example_complaint_letter.jpg")
+        print("Created dummy image")
+
+    except ImportError:
+        print("PIL not available, skipping image creation")
+
+
+def main():
+    """Main setup function."""
+    print("Setting up test data for GitHub Actions...")
+    print(f"Current working directory: {os.getcwd()}")
+    print(f"Python version: {sys.version}")
+
+    create_directories()
+    create_dummy_pdf()
+    create_dummy_csv()
+    create_dummy_word_doc()
+    create_allow_deny_lists()
+    create_ocr_output()
+    create_dummy_image()
+
+    print("\nTest data setup complete!")
+    print("Created files:")
+    for root, dirs, files in os.walk("doc_redaction/example_data"):
+        for file in files:
+            file_path = os.path.join(root, file)
+            print(f"  {file_path}")
+            # Verify the file exists and has content
+            if os.path.exists(file_path):
+                file_size = os.path.getsize(file_path)
+                print(f"    Size: {file_size} bytes")
+            else:
+                print("    WARNING: File does not exist!")
+
+    # Verify critical files exist
+    critical_files = [
+        "doc_redaction/example_data/Partnership-Agreement-Toolkit_0_0.pdf",
+        "doc_redaction/example_data/graduate-job-example-cover-letter.pdf",
+        "doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf",
+    ]
+
+    print("\nVerifying critical test files:")
+    for file_path in critical_files:
+        if os.path.exists(file_path):
+            file_size = os.path.getsize(file_path)
+            print(f"✅ {file_path} exists ({file_size} bytes)")
+        else:
+            print(f"❌ {file_path} MISSING!")
+
+
+if __name__ == "__main__":
+    main()

.github/workflow_README.md

@@ -0,0 +1,183 @@
+# GitHub Actions CI/CD Setup
+
+This directory contains GitHub Actions workflows for automated testing of the CLI redaction application.
+
+## Workflows Overview
+
+### 1. **Simple Test Run** (`.github/workflows/simple-test.yml`)
+- **Purpose**: Basic test execution
+- **Triggers**: Push to main/dev, Pull requests
+- **OS**: Ubuntu Latest
+- **Python**: 3.11
+- **Features**: 
+  - Installs system dependencies
+  - Sets up test data
+  - Runs CLI tests
+  - Runs pytest
+
+### 2. **Comprehensive CI/CD** (`.github/workflows/ci.yml`)
+- **Purpose**: Full CI/CD pipeline
+- **Features**:
+  - Linting (Ruff, Black)
+  - Unit tests (Python 3.10, 3.11, 3.12)
+  - Integration tests
+  - Security scanning (Safety, Bandit)
+  - Coverage reporting
+  - Package building (on main branch)
+
+### 3. **Multi-OS Testing** (`.github/workflows/multi-os-test.yml`)
+- **Purpose**: Cross-platform testing
+- **OS**: Ubuntu, macOS (Windows not included currently but may be reintroduced)
+- **Python**: 3.10, 3.11, 3.12
+- **Features**: Tests compatibility across different operating systems
+
+### 4. **Basic Test Suite** (`.github/workflows/test.yml`)
+- **Purpose**: Original test workflow
+- **Features**: 
+  - Multiple Python versions
+  - System dependency installation
+  - Test data creation
+  - Coverage reporting
+
+## Setup Scripts
+
+### Test Data Setup (`.github/scripts/setup_test_data.py`)
+Creates dummy test files when example data is not available:
+- PDF documents
+- CSV files
+- Word documents
+- Images
+- Allow/deny lists
+- OCR output files
+
+## Usage
+
+### Running Tests Locally
+
+```bash
+# Install dependencies
+pip install -r requirements.txt
+pip install pytest pytest-cov
+
+# Setup test data
+python .github/scripts/setup_test_data.py
+
+# Run tests
+cd test
+python cli_epilog_suite.py
+```
+
+### GitHub Actions Triggers
+
+1. **Push to main/dev**: Runs all tests
+2. **Pull Request**: Runs tests and linting
+3. **Daily Schedule**: Runs tests at 2 AM UTC
+4. **Manual Trigger**: Can be triggered manually from GitHub
+
+## Configuration
+
+### Environment Variables
+- `PYTHON_VERSION`: Default Python version (3.11)
+- `PYTHONPATH`: Set automatically for test discovery
+
+### Caching
+- Pip dependencies are cached for faster builds
+- Cache key based on requirements.txt hash
+
+### Artifacts
+- Test results (JUnit XML)
+- Coverage reports (HTML, XML)
+- Security reports
+- Build artifacts (on main branch)
+
+## Test Data
+
+The workflows automatically create test data when example files are missing:
+
+### Required Files Created:
+- `example_data/example_of_emails_sent_to_a_professor_before_applying.pdf`
+- `example_data/combined_case_notes.csv`
+- `example_data/Bold minimalist professional cover letter.docx`
+- `example_data/example_complaint_letter.jpg`
+- `example_data/test_allow_list_*.csv`
+- `example_data/partnership_toolkit_redact_*.csv`
+- `example_data/example_outputs/doubled_output_joined.pdf_ocr_output.csv`
+
+### Dependencies Installed:
+- **System**: tesseract-ocr, poppler-utils, OpenGL libraries
+- **Python**: All requirements.txt packages + pytest, reportlab, pillow
+
+## Workflow Status
+
+### Success Criteria:
+- ✅ All tests pass
+- ✅ No linting errors
+- ✅ Security checks pass
+- ✅ Coverage meets threshold (if configured)
+
+### Failure Handling:
+- Tests are designed to skip gracefully if files are missing
+- AWS tests are expected to fail without credentials
+- System dependency failures are handled with fallbacks
+
+## Customization
+
+### Adding New Tests:
+1. Add test methods to `test/cli_epilog_suite.py` or pytest files under `test/test_*.py`
+2. Update test data in `setup_test_data.py` if needed
+3. Tests will automatically run in all workflows
+
+### Modifying Workflows:
+1. Edit the appropriate `.yml` file
+2. Test locally first
+3. Push to trigger the workflow
+
+### Environment-Specific Settings:
+- **Ubuntu**: Full system dependencies
+- **Windows**: Python packages only
+- **macOS**: Homebrew dependencies
+
+## Troubleshooting
+
+### Common Issues:
+
+1. **Missing Dependencies**:
+   - Check system dependency installation
+   - Verify Python package versions
+
+2. **Test Failures**:
+   - Check test data creation
+   - Verify file paths
+   - Review test output logs
+
+3. **AWS Test Failures**:
+   - Expected without credentials
+   - Tests are designed to handle this gracefully
+
+4. **System Dependency Issues**:
+   - Different OS have different requirements
+   - Check the specific OS section in workflows
+
+### Debug Mode:
+Add `--verbose` or `-v` flags to pytest commands for more detailed output.
+
+## Security
+
+- Dependencies are scanned with Safety
+- Code is scanned with Bandit
+- No secrets are exposed in logs
+- Test data is temporary and cleaned up
+
+## Performance
+
+- Tests run in parallel where possible
+- Dependencies are cached
+- Only necessary system packages are installed
+- Test data is created efficiently
+
+## Monitoring
+
+- Workflow status is visible in GitHub Actions tab
+- Coverage reports are uploaded to Codecov
+- Test results are available as artifacts
+- Security reports are generated and stored

.github/workflows/archive_workflows/multi-os-test.yml

@@ -0,0 +1,115 @@
+name: Multi-OS Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    env:
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest] # windows-latest, not included as tesseract cannot be installed silently
+        python-version: ["3.11", "3.12", "3.13"]
+        exclude:
+          # Exclude some combinations to reduce CI time
+          #- os: windows-latest
+          #  python-version: ["3.12", "3.13"]
+          - os: macos-latest
+            python-version: ["3.12", "3.13"]
+
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Install system dependencies (Ubuntu)
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Install system dependencies (macOS)
+      if: matrix.os == 'macos-latest'
+      run: |
+        brew install tesseract poppler
+    
+    - name: Install system dependencies (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        # Create tools directory
+        if (!(Test-Path "C:\tools")) {
+            mkdir C:\tools
+        }
+        
+        # Download and install Tesseract
+        $tesseractUrl = "https://github.com/tesseract-ocr/tesseract/releases/download/5.5.0/tesseract-ocr-w64-setup-5.5.0.20241111.exe"
+        $tesseractInstaller = "C:\tools\tesseract-installer.exe"
+        Invoke-WebRequest -Uri $tesseractUrl -OutFile $tesseractInstaller
+        
+        # Install Tesseract silently
+        Start-Process -FilePath $tesseractInstaller -ArgumentList "/S", "/D=C:\tools\tesseract" -Wait
+        
+        # Download and extract Poppler
+        $popplerUrl = "https://github.com/oschwartz10612/poppler-windows/releases/download/v25.07.0-0/Release-25.07.0-0.zip"
+        $popplerZip = "C:\tools\poppler.zip"
+        Invoke-WebRequest -Uri $popplerUrl -OutFile $popplerZip
+        
+        # Extract Poppler
+        Expand-Archive -Path $popplerZip -DestinationPath C:\tools\poppler -Force
+        
+        # Add to PATH
+        echo "C:\tools\tesseract" >> $env:GITHUB_PATH
+        echo "C:\tools\poppler\poppler-25.07.0\Library\bin" >> $env:GITHUB_PATH
+        
+        # Set environment variables for your application
+        echo "TESSERACT_FOLDER=C:\tools\tesseract" >> $env:GITHUB_ENV
+        echo "POPPLER_FOLDER=C:\tools\poppler\poppler-25.07.0\Library\bin" >> $env:GITHUB_ENV
+        echo "TESSERACT_DATA_FOLDER=C:\tools\tesseract\tessdata" >> $env:GITHUB_ENV
+        
+        # Verify installation using full paths (since PATH won't be updated in current session)
+        & "C:\tools\tesseract\tesseract.exe" --version
+        & "C:\tools\poppler\poppler-25.07.0\Library\bin\pdftoppm.exe" -v
+    
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pytest pytest-cov reportlab pillow
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+    
+    - name: Run CLI tests
+      run: |
+        cd test
+        python cli_epilog_suite.py
+    
+    - name: Run tests with pytest
+      run: |
+        pytest test/ -v --tb=short

.github/workflows/ci.yml

@@ -0,0 +1,268 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  #schedule:
+  # Run tests daily at 2 AM UTC
+  #  - cron: '0 2 * * *'
+
+permissions:
+  contents: read
+  actions: read
+  pull-requests: write
+  issues: write
+
+env:
+  PYTHON_VERSION: "3.11"
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install ruff black
+    
+    - name: Run Ruff linter
+      run: ruff check .
+    
+    - name: Run Black formatter check
+      run: black --check .
+
+  test-unit:
+    runs-on: ubuntu-latest
+    env:
+      # Avoid optional VLM/torch import path in tools.run_vlm (not installed in lightweight CI deps)
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    strategy:
+      matrix:
+        python-version: [3.11, 3.12, 3.13]
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Cache pip dependencies
+      uses: actions/cache@v5
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements_lightweight.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_lightweight.txt
+        pip install pytest pytest-cov pytest-html pytest-xdist reportlab pillow
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+        echo "Setup script completed. Checking results:"
+        ls -la doc_redaction/example_data/ || echo "doc_redaction/example_data directory not found"
+    
+    - name: Verify test data files
+      run: |
+        echo "Checking if critical test files exist:"
+        ls -la doc_redaction/example_data/
+        echo "Checking for specific PDF files:"
+        ls -la doc_redaction/example_data/*.pdf || echo "No PDF files found"
+        echo "Checking file sizes:"
+        find doc_redaction/example_data -name "*.pdf" -exec ls -lh {} \;
+    
+    - name: Clean up problematic config files
+      run: |
+        rm -f config*.py || true
+    
+    - name: Run CLI tests
+      run: |
+        cd test
+        python cli_epilog_suite.py
+    
+    - name: Run tests with pytest (JUnit and coverage)
+      run: |
+        pytest test/ -v --tb=short \
+          --junitxml=test-results.xml \
+          --cov=. --cov-config=.coveragerc \
+          --cov-report=xml --cov-report=html --cov-report=term
+    
+    #- name: Upload coverage to Codecov - not necessary
+    #  uses: codecov/codecov-action@v3
+    #  if: matrix.python-version == '3.11'
+    #  with:
+    #    file: ./coverage.xml
+    #    flags: unittests
+    #    name: codecov-umbrella
+    #    fail_ci_if_error: false
+    
+    - name: Upload test results
+      uses: actions/upload-artifact@v6
+      if: always()
+      with:
+        name: test-results-python-${{ matrix.python-version }}
+        path: |
+          test-results.xml
+          htmlcov/
+          coverage.xml
+
+  test-integration:
+    runs-on: ubuntu-latest
+    needs: [lint, test-unit]
+    env:
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_lightweight.txt
+        pip install pytest pytest-cov reportlab pillow
+    
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+        echo "Setup script completed. Checking results:"
+        ls -la doc_redaction/example_data/ || echo "doc_redaction/example_data directory not found"
+    
+    - name: Verify test data files
+      run: |
+        echo "Checking if critical test files exist:"
+        ls -la doc_redaction/example_data/
+        echo "Checking for specific PDF files:"
+        ls -la doc_redaction/example_data/*.pdf || echo "No PDF files found"
+        echo "Checking file sizes:"
+        find doc_redaction/example_data -name "*.pdf" -exec ls -lh {} \;
+    
+    - name: Run integration tests
+      run: |
+        cd test
+        python demo_single_test.py
+    
+    - name: Test CLI help
+      run: |
+        python cli_redact.py --help
+    
+    - name: Test CLI version
+      run: |
+        python -c "import sys; print(f'Python {sys.version}')"
+
+  security:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install safety bandit
+    
+    #- name: Run safety scan - removed as now requires login
+    #  run: |
+    #    safety scan -r requirements.txt
+    
+    - name: Run bandit security check
+      run: |
+        bandit -r . -f json -o bandit-report.json || true
+    
+    - name: Upload security report
+      uses: actions/upload-artifact@v6
+      if: always()
+      with:
+        name: security-report
+        path: bandit-report.json
+
+  build:
+    runs-on: ubuntu-latest
+    needs: [lint, test-unit]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+    
+    - name: Build package
+      run: |
+        python -m build
+    
+    - name: Check package
+      run: |
+        twine check dist/*
+    
+    - name: Upload build artifacts
+      uses: actions/upload-artifact@v6
+      with:
+        name: dist
+        path: dist/

.github/workflows/simple-test.yml

@@ -0,0 +1,73 @@
+name: Simple Test Run
+
+on:
+  push:
+    branches: [ dev ]
+  pull_request:
+    branches: [ dev ]
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      SHOW_VLM_MODEL_OPTIONS: "False"
+    
+    steps:
+    - uses: actions/checkout@v6
+    
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.12"
+    
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          tesseract-ocr \
+          tesseract-ocr-eng \
+          poppler-utils \
+          libgl1-mesa-dri \
+          libglib2.0-0 \
+          libsm6 \
+          libxext6 \
+          libxrender-dev \
+          libgomp1
+    
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_lightweight.txt
+        pip install pytest pytest-cov reportlab pillow
+    
+    - name: Download spaCy model
+      run: |
+        python -m spacy download en_core_web_lg
+    
+    - name: Setup test data
+      run: |
+        python .github/scripts/setup_test_data.py
+        echo "Setup script completed. Checking results:"
+        ls -la doc_redaction/example_data/ || echo "doc_redaction/example_data directory not found"
+    
+    - name: Verify test data files
+      run: |
+        echo "Checking if critical test files exist:"
+        ls -la doc_redaction/example_data/
+        echo "Checking for specific PDF files:"
+        ls -la doc_redaction/example_data/*.pdf || echo "No PDF files found"
+        echo "Checking file sizes:"
+        find doc_redaction/example_data -name "*.pdf" -exec ls -lh {} \;
+    
+    - name: Run CLI tests
+      run: |
+        cd test
+        python cli_epilog_suite.py
+    
+    - name: Run tests with pytest
+      run: |
+        pytest test/ -v --tb=short

.github/workflows/sync_to_hf.yml

@@ -0,0 +1,53 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [dev]
+
+permissions:
+  contents: read
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 1      # Only get the latest state
+          lfs: true           # Download actual LFS files so they can be pushed
+
+      - name: Install Git LFS
+        run: git lfs install
+
+      - name: Recreate repo history (single-commit force push)
+        run: |
+          # 1. Capture the message BEFORE we delete the .git folder
+          COMMIT_MSG=$(git log -1 --pretty=%B)
+          echo "Syncing commit message: $COMMIT_MSG"
+
+          # 2. DELETE the .git folder. 
+          # This turns the repo into a standard folder of files.
+          rm -rf .git
+
+          # 3. Re-initialize a brand new git repo
+          git init -b main
+          git config --global user.name "$HF_USERNAME"
+          git config --global user.email "$HF_EMAIL"
+          
+          # 4. Re-install LFS (needs to be done after git init)
+          git lfs install
+          
+          # 5. Add the remote
+          git remote add hf https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$HF_REPO_ID
+          
+          # 6. Add all files
+          # Since this is a fresh init, Git sees EVERY file as "New"
+          git add .
+          
+          # 7. Commit and Force Push
+          git commit -m "Sync: $COMMIT_MSG"
+          git push --force hf main
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+          HF_EMAIL: ${{ secrets.HF_EMAIL }}
+          HF_REPO_ID: ${{ secrets.HF_REPO_ID }}

.github/workflows/sync_to_hf_zero_gpu.yml

@@ -0,0 +1,58 @@
+name: Sync to Hugging Face hub Zero GPU
+on:
+  push:
+    branches: [dev]
+
+permissions:
+  contents: read
+
+jobs:
+  sync-to-hub-zero-gpu:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 1      # Only get the latest state
+          lfs: true           # Download actual LFS files so they can be pushed
+
+      - name: Install Git LFS
+        run: git lfs install
+
+      # HF Spaces read Space config from README.md front matter. The repo README
+      # targets GitHub (e.g. docker); patch only this CI checkout before HF push.
+      - name: Apply HF Zero GPU Space README front matter
+        run: python3 tools/apply_hf_zero_gpu_readme_frontmatter.py
+
+      - name: Recreate repo history (single-commit force push)
+        run: |
+          # 1. Capture the message BEFORE we delete the .git folder
+          COMMIT_MSG=$(git log -1 --pretty=%B)
+          echo "Syncing commit message: $COMMIT_MSG"
+
+          # 2. DELETE the .git folder. 
+          # This turns the repo into a standard folder of files.
+          rm -rf .git
+
+          # 3. Re-initialize a brand new git repo
+          git init -b main
+          git config --global user.name "$HF_USERNAME"
+          git config --global user.email "$HF_EMAIL"
+          
+          # 4. Re-install LFS (needs to be done after git init)
+          git lfs install
+          
+          # 5. Add the remote
+          git remote add hf https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$HF_REPO_ID_ZERO_GPU
+          
+          # 6. Add all files
+          # Since this is a fresh init, Git sees EVERY file as "New"
+          git add .
+          
+          # 7. Commit and Force Push
+          git commit -m "Sync: $COMMIT_MSG"
+          git push --force hf main
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+          HF_EMAIL: ${{ secrets.HF_EMAIL }}
+          HF_REPO_ID_ZERO_GPU: ${{ secrets.HF_REPO_ID_ZERO_GPU }}

.gitignore

@@ -0,0 +1,54 @@
+*.url
+*.ipynb
+*.pyc
+*.qmd
+_quarto.yml
+quarto_site/*
+src/*
+redaction_deps/*
+.venv/*
+examples/*
+processing/*
+input/*
+output/*
+tools/__pycache__/*
+old_code/*
+tesseract/*
+poppler/*
+build/*
+dist/*
+build_deps/*
+logs/*
+usage/*
+feedback/*
+config/*
+user_guide/*
+_extensions/*
+doc_redaction.egg-info/*
+.venv_pypi_test/*
+cdk/config/*
+cdk/cdk.out/*
+cdk/archive/*
+tld/*
+tmp/*
+docs/*
+cdk.out/*
+cdk.json
+cdk.context.json
+.quarto/*
+/.quarto/
+/_site/
+test/config/*
+test/feedback/*
+test/input/*
+test/logs/*
+test/output/*
+test/tmp/*
+test/usage/*
+.ruff_cache/*
+model_cache/*
+sanitized_file/*
+src/doc_redaction.egg-info/*
+docker_compose/*
+**/*.quarto_ipynb
+skills/example_prompts/*

AGENTS.md

@@ -0,0 +1,97 @@
+# AGENTS.md
+
+Context for AI coding agents working on **doc_redaction** (PII redaction for PDFs, images, Word, and tabular files). Human-oriented docs: [README.md](README.md). User guide: [doc_redaction user guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html).
+
+## Project overview
+
+- **Stack**: Python 3.10+, Gradio UI ([app.py](app.py)), optional FastAPI when `RUN_FASTAPI` is enabled, AWS/LLM integrations via [tools/config.py](tools/config.py) and env files under `config/`.
+- **License**: AGPL-3.0-only (see [pyproject.toml](pyproject.toml)). Respect license terms when adding dependencies.
+- **Accuracy**: Outputs are not guaranteed complete; downstream use should assume **human review** of redacted material.
+
+## Cursor skills: redaction workflow (optional)
+
+For agents operating the deployed app (Gradio Client, review CSV, `/review_apply`), these repo-local playbooks are a suggested ladder:
+
+1. **[`skills/doc-redaction-app/SKILL.md`](skills/doc-redaction-app/SKILL.md)** — first-pass redaction (`/doc_redact` / `/redact_document`) and downloading artifacts.
+2. **[`skills/doc-redact-page-review/SKILL.md`](skills/doc-redact-page-review/SKILL.md)** — after outputs exist: **parallel per-page** child agents, merge into one full-document `*_review_file.csv`, **single** `/review_apply` from the parent.
+3. **[`skills/doc-redaction-modifications/SKILL.md`](skills/doc-redaction-modifications/SKILL.md)** — CSV mechanics, `preview_redaction_boxes`, `/review_apply` patterns, verification, VLM and PyMuPDF fallbacks (single-thread edits and the **technical** reference for page-review children).
+
+## Setup
+
+1. **System**: Install **Tesseract** and **Poppler** (required for OCR/PDF). See [README.md](README.md) (Windows/Linux sections).
+2. **Python**: Create a venv, then install the project (e.g. `pip install -e ".[dev]"` or follow README).
+3. **Configuration**: Copy or edit environment/config as described in README / `config/` (e.g. `app_config.env`). Do not commit secrets.
+
+## Run locally
+
+- Gradio/FastAPI entrypoint is [app.py](app.py). With FastAPI enabled, typical pattern is `uvicorn app:app --host 0.0.0.0 --port 7860` (exact host/port from your config).
+- OpenAPI docs: `/docs` when the FastAPI app is mounted.
+
+## Tests
+
+- Run from repo root: `pytest` (optional: `pytest test/`).
+- Fix failures related to your changes before opening a PR.
+
+## Line order (local OCR and simple text extraction)
+
+Multi-column layouts use shared logic in [`tools/ocr_reading_order.py`](tools/ocr_reading_order.py). Controlled by **`LOCAL_OCR_READING_ORDER`** (`column` default, `legacy` for previous top-left behaviour).
+
+### Local OCR (Paddle/Tesseract)
+
+Word boxes are merged into line-level CSV rows in [`combine_ocr_results`](tools/custom_image_analyser_engine.py).
+
+- **`column`**: detect text columns, assign line numbers down each column left-to-right; full-width lines (headers) first. Stops cross-column merging that produced wide erroneous lines on multi-column PDFs. **Auto-fallback**: if no consecutive horizontal gutter is found on a text row (`OCR_COLUMN_GUTTER_MIN_FRACTION`, default `0.04`), the page is treated as single-column and ordered by `(top, left)` (e.g. business letters).
+- **`PADDLE_PRESERVE_LINE_BOXES=True`** or **`CONVERT_LINE_TO_WORD_LEVEL=False`** with Paddle: keep Paddle line boxes (skip word split + regrouping); line numbers still use column reading order.
+
+### Simple text extraction (PyMuPDF)
+
+[`redact_text_pdf`](tools/file_redaction.py) → [`process_page_to_structured_ocr_pymupdf`](tools/file_redaction.py) calls [`reorder_structured_text_lines`](tools/ocr_reading_order.py) after collecting lines, using **`page.mediabox`** width/height for full-span header detection. Each PDF line keeps its bbox; only order and `line` indices change.
+
+### Tunables (both routes)
+
+`OCR_FULL_SPAN_WIDTH_RATIO`, `OCR_COLUMN_GAP_MIN_FRACTION`, `OCR_COLUMN_GUTTER_MIN_FRACTION`, `OCR_LINE_Y_THRESHOLD_FRACTION`, `OCR_LINE_Y_THRESHOLD_MIN_PX`.
+
+Changing line order affects PII page text, duplicate-page detection, and review CSV line indices on multi-column documents; re-review after upgrading.
+
+## Agentic / programmatic access (two surfaces)
+
+### 1. FastAPI Agent API (recommended for LLM agents: small JSON bodies)
+
+When `RUN_FASTAPI` is true, routes are mounted under **`/agent`** ([agent_routes.py](agent_routes.py)).
+
+- **Catalog**: `GET /agent/operations` — maps each Gradio `api_name` to an HTTP path and notes whether the route is implemented via CLI or returns HTTP 501 for Gradio-only flows.
+- **Implemented POST routes** (CLI- or [tools/simplified_api.py](tools/simplified_api.py)-backed where noted):  
+  `redact_document`, `redact_data`, `find_duplicate_pages`, `find_duplicate_tabular`, `summarise_document`, `combine_review_pdfs`, `combine_review_csvs`, `export_review_redaction_overlay`, `export_review_page_ocr_visualisation`, `apply_review_redactions` (`apply_review_redactions`: JSON body `pdf_path`, `review_csv_path` basename must contain `_review_file`, optional `output_dir` / `input_dir` / `text_extract_method` / `efficient_ocr`).  
+  Note: on Gradio ([app.py](app.py)), the Review-tab visual exports use `api_name` **`page_redaction_review_image`** and **`page_ocr_review_image`**; the **`/agent`** routes above keep the explicit `export_review_*` names for the same operations.
+- **Gradio-only stubs** (501 + JSON hint): `load_and_prepare_documents_or_data`, `word_level_ocr_text_search`.
+- **Auth**: If `AGENT_API_KEY` is set in the environment, send header `X-Agent-API-Key` with that value.
+- **Paths**: Inputs must resolve to files under the repo root, `INPUT_FOLDER`, or `OUTPUT_FOLDER` (see router validation).
+
+Implementation uses **`cli_redact.main(direct_mode_args=...)`** where a CLI task exists (same behaviour as [cli_redact.py](cli_redact.py)); `apply_review_redactions` calls [tools/simplified_api.py](tools/simplified_api.py) instead.
+
+### 2. Gradio Client API (e.g. Hugging Face Spaces)
+
+For remote Spaces or any Gradio deployment exposing the HTTP API:
+
+- **Schema**: `GET https://<host>/gradio_api/info`
+- **Call**: `POST https://<host>/gradio_api/call/{api_name}` with body `{"data":[...]}` (argument order matches the named endpoint’s component list).
+- **Poll**: `GET https://<host>/gradio_api/call/{api_name}/{event_id}`
+- **Hugging Face**: `Authorization: Bearer $HF_TOKEN`
+
+Named `api_name` values in this app include: `redact_document`, `load_and_prepare_documents_or_data`, `apply_review_redactions`, **`doc_redact`** (simple `gr.api`: one PDF/image + optional OCR/PII knobs; returns `(output_paths, message)`; `api_name='/doc_redact'`), **`review_apply`** (simple `gr.api`: PDF + `*_review_file.csv`; returns `(output_paths, message)`; `api_name='/review_apply'`), **`preview_boxes`** (simple `gr.api`: PDF + `*_review_file.csv`; renders proposed boxes onto the original PDF and returns `(zip_path, message)` — use to verify coordinates *before* calling `review_apply`, no redaction applied; `api_name='/preview_boxes'`), **`pdf_summarise`** (simple `gr.api`: PDF + optional summarisation/OCR knobs; returns `(output_paths, status_message, summary_text)`; `api_name='/pdf_summarise'`), **`tabular_redact`** (simple `gr.api`: one tabular file (CSV/XLSX/Parquet/DOCX) + optional knobs; returns `(output_paths, message)`; `api_name='/tabular_redact'`), **`page_redaction_review_image`** (short review overlay export; `api_name='/page_redaction_review_image'`), **`page_ocr_review_image`** (short OCR visualisation export; `api_name='/page_ocr_review_image'`), `word_level_ocr_text_search`, `redact_data`, `find_duplicate_pages`, `find_duplicate_tabular`, `summarise_document`, `combine_review_csvs`, `combine_review_pdfs`. The matching **`POST /agent`** names for those two visual exports are `export_review_redaction_overlay` and `export_review_page_ocr_visualisation` (§1). Many endpoints require **many positional arguments** (full Gradio state); prefer the short `gr.api` routes above or **`POST /agent/apply_review_redactions`** where applicable instead of building the full `data` array from `/gradio_api/info`.
+
+## CLI parity
+
+For scripting and tests, `python cli_redact.py` with flags is authoritative; programmatic merges use `get_cli_default_args_dict()` in [cli_redact.py](cli_redact.py).
+
+## Security and data handling
+
+- Do not commit API keys, tokens, or customer data.
+- Treat paths as untrusted outside validated roots (see [tools/secure_path_utils.py](tools/secure_path_utils.py)).
+- Optional `instruction` / LLM fields must not be passed into shell or unconstrained config keys.
+
+## Conventions for PRs
+
+- Keep changes focused; avoid drive-by refactors.
+- Match existing naming and patterns in [app.py](app.py) and [tools/](tools/).
+- Update tests when behaviour changes; run `pytest` before merge.

Dockerfile

@@ -0,0 +1,232 @@
+# Stage 1: Build dependencies and download models
+FROM public.ecr.aws/docker/library/python:3.12.13-slim-trixie AS builder
+
+# Install system dependencies
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y --no-install-recommends \
+        g++ \
+        make \
+        cmake \
+        unzip \
+        libcurl4-openssl-dev \
+        git \
+    && pip install --upgrade pip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /src
+
+COPY requirements_lightweight.txt .
+
+RUN pip install --verbose --no-cache-dir --target=/install -r requirements_lightweight.txt && rm requirements_lightweight.txt
+
+ARG INSTALL_GRADIO_MCP=False
+ENV INSTALL_GRADIO_MCP=${INSTALL_GRADIO_MCP}
+
+RUN if [ "$INSTALL_GRADIO_MCP" = "True" ]; then \
+    pip install --verbose --no-cache-dir --force-reinstall --target=/install "gradio[mcp]<=6.10.0"; \
+fi
+
+# Optionally install PaddleOCR if the INSTALL_PADDLEOCR environment variable is set to True. Note that GPU-enabled PaddleOCR is unlikely to work in the same environment as a GPU-enabled version of PyTorch, so it is recommended to install PaddleOCR as a CPU-only version if you want to use GPU-enabled PyTorch.
+
+ARG INSTALL_PADDLEOCR=False
+ENV INSTALL_PADDLEOCR=${INSTALL_PADDLEOCR}
+
+ARG PADDLE_GPU_ENABLED=False
+ENV PADDLE_GPU_ENABLED=${PADDLE_GPU_ENABLED}
+
+RUN if [ "$INSTALL_PADDLEOCR" = "True" ] && [ "$PADDLE_GPU_ENABLED" = "False" ]; then \
+    pip install --verbose --no-cache-dir --target=/install "protobuf<=7.34.0" && \
+    pip install --verbose --no-cache-dir --target=/install "paddlepaddle<=3.2.1" && \
+    pip install --verbose --no-cache-dir --target=/install "paddleocr<=3.3.0"; \
+elif [ "$INSTALL_PADDLEOCR" = "True" ] && [ "$PADDLE_GPU_ENABLED" = "True" ]; then \
+    pip install --verbose --no-cache-dir --target=/install "protobuf<=7.34.0" && \
+    pip install --verbose --no-cache-dir --target=/install "paddlepaddle-gpu<=3.2.1" --index-url https://www.paddlepaddle.org.cn/packages/stable/cu129/ && \
+    pip install --verbose --no-cache-dir --target=/install "paddleocr<=3.3.0"; \
+fi
+
+ARG INSTALL_VLM=False
+ENV INSTALL_VLM=${INSTALL_VLM}
+
+ARG TORCH_GPU_ENABLED=False
+ENV TORCH_GPU_ENABLED=${TORCH_GPU_ENABLED}
+
+# Optionally install VLM/LLM packages if the INSTALL_VLM environment variable is set to True.
+RUN if [ "$INSTALL_VLM" = "True" ] && [ "$TORCH_GPU_ENABLED" = "False" ]; then \
+    pip install --verbose --no-cache-dir --target=/install \
+    "torch==2.9.1+cpu" \
+    "torchvision==0.24.1+cpu" \
+    "transformers<=5.5.4" \
+    "accelerate<=1.13.0" \
+    "bitsandbytes<=0.49.2" \
+    "sentencepiece<=0.2.1" \
+    --extra-index-url https://download.pytorch.org/whl/cpu; \
+elif [ "$INSTALL_VLM" = "True" ] && [ "$TORCH_GPU_ENABLED" = "True" ]; then \
+    pip install --verbose --no-cache-dir --target=/install "torch<=2.8.0" --index-url https://download.pytorch.org/whl/cu129 && \
+    pip install --verbose --no-cache-dir --target=/install "torchvision<=0.23.0" --index-url https://download.pytorch.org/whl/cu129 && \
+    pip install --verbose --no-cache-dir --target=/install \
+        "transformers<=5.5.4" \
+        "accelerate<=1.13.0" \
+        "bitsandbytes<=0.49.2" \
+        "sentencepiece<=0.2.1" && \
+    pip install --verbose --no-cache-dir --target=/install "optimum<=2.1.0" && \
+    pip install --verbose --no-cache-dir --target=/install  https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.8cxx11abiTRUE-cp312-cp312-linux_x86_64.whl && \
+    pip install --verbose --no-cache-dir --target=/install  https://github.com/ModelCloud/GPTQModel/releases/download/v5.8.0/gptqmodel-5.8.0+cu128torch2.8-cp312-cp312-linux_x86_64.whl; \
+fi
+
+# ===================================================================
+# Stage 2: A common base for both Lambda and Gradio
+# ===================================================================
+FROM public.ecr.aws/docker/library/python:3.12.13-slim-trixie AS base
+
+# MUST re-declare ARGs in every stage where they are used in RUN commands
+ARG TORCH_GPU_ENABLED=False
+ARG PADDLE_GPU_ENABLED=False
+
+ENV TORCH_GPU_ENABLED=${TORCH_GPU_ENABLED}
+ENV PADDLE_GPU_ENABLED=${PADDLE_GPU_ENABLED}
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    tesseract-ocr \
+    poppler-utils \
+    libgl1 \
+    libglib2.0-0 && \
+    if [ "$TORCH_GPU_ENABLED" = "True" ] || [ "$PADDLE_GPU_ENABLED" = "True" ]; then \
+        apt-get install -y --no-install-recommends libgomp1; \
+    fi && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV APP_HOME=/home/user
+
+# Set env variables for Gradio & other apps
+ENV GRADIO_TEMP_DIR=/tmp/gradio_tmp/ \
+    MPLCONFIGDIR=/tmp/matplotlib_cache/ \
+    GRADIO_OUTPUT_FOLDER=$APP_HOME/app/output/ \
+    GRADIO_INPUT_FOLDER=$APP_HOME/app/input/ \
+    FEEDBACK_LOGS_FOLDER=$APP_HOME/app/feedback/ \
+    ACCESS_LOGS_FOLDER=$APP_HOME/app/logs/ \
+    USAGE_LOGS_FOLDER=$APP_HOME/app/usage/ \
+    CONFIG_FOLDER=$APP_HOME/app/config/ \
+    XDG_CACHE_HOME=/tmp/xdg_cache/user_1000 \
+    TESSERACT_DATA_FOLDER=/usr/share/tessdata \
+    GRADIO_SERVER_NAME=0.0.0.0 \
+    GRADIO_SERVER_PORT=7860 \
+    PATH=$APP_HOME/.local/bin:$PATH \
+    PYTHONPATH=$APP_HOME/app \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    GRADIO_ALLOW_FLAGGING=never \
+    GRADIO_NUM_PORTS=1 \    
+    GRADIO_ANALYTICS_ENABLED=False
+
+# Copy Python packages from the builder stage
+COPY --from=builder /install /usr/local/lib/python3.12/site-packages/
+COPY --from=builder /install/bin /usr/local/bin/
+
+# Reinstall protobuf into the final site-packages. Builder uses multiple `pip install --target=/install`
+# passes; that can break the `google` namespace so `google.protobuf` is missing and Paddle fails at import.
+RUN pip install --no-cache-dir "protobuf<=7.34.0"
+
+# English pipeline is not a normal PyPI dependency; bundle it in the image so runtime works offline.
+# Placed before COPY app code so application changes do not invalidate this layer.
+RUN python -m spacy download en_core_web_lg
+
+# Copy your application code and entrypoint
+COPY . ${APP_HOME}/app
+COPY entrypoint.sh ${APP_HOME}/app/entrypoint.sh
+# Fix line endings and set execute permissions
+RUN sed -i 's/\r$//' ${APP_HOME}/app/entrypoint.sh \
+    && chmod +x ${APP_HOME}/app/entrypoint.sh
+
+WORKDIR ${APP_HOME}/app
+
+# ===================================================================
+# FINAL Stage 3: The Lambda Image (runs as root for simplicity)
+# ===================================================================
+FROM base AS lambda
+# Set runtime ENV for Lambda mode
+ENV APP_MODE=lambda
+ENTRYPOINT ["/home/user/app/entrypoint.sh"]
+CMD ["lambda_entrypoint.lambda_handler"]
+
+# ===================================================================
+# FINAL Stage 4: The Gradio Image (runs as a secure, non-root user)
+# ===================================================================
+FROM base AS gradio
+# Set runtime ENV for Gradio mode
+ENV APP_MODE=gradio
+
+# Create non-root user
+RUN useradd -m -u 1000 user
+
+# Create the base application directory and set its ownership
+RUN mkdir -p ${APP_HOME}/app && chown user:user ${APP_HOME}/app
+
+# Create required sub-folders within the app directory and set their permissions
+# This ensures these specific directories are owned by 'user'
+RUN mkdir -p \
+    ${APP_HOME}/app/output \
+    ${APP_HOME}/app/input \
+    ${APP_HOME}/app/logs \
+    ${APP_HOME}/app/usage \
+    ${APP_HOME}/app/feedback \
+    ${APP_HOME}/app/config \
+    && chown user:user \
+    ${APP_HOME}/app/output \
+    ${APP_HOME}/app/input \
+    ${APP_HOME}/app/logs \
+    ${APP_HOME}/app/usage \
+    ${APP_HOME}/app/feedback \
+    ${APP_HOME}/app/config \
+    && chmod 755 \
+    ${APP_HOME}/app/output \
+    ${APP_HOME}/app/input \
+    ${APP_HOME}/app/logs \
+    ${APP_HOME}/app/usage \
+    ${APP_HOME}/app/feedback \
+    ${APP_HOME}/app/config 
+
+# Now handle the /tmp and /var/tmp directories and their subdirectories, paddle, spacy, tessdata
+RUN mkdir -p /tmp/gradio_tmp /tmp/tld /tmp/matplotlib_cache /tmp /var/tmp ${XDG_CACHE_HOME} \
+    && chown user:user /tmp /var/tmp /tmp/gradio_tmp /tmp/tld /tmp/matplotlib_cache ${XDG_CACHE_HOME} \
+    && chmod 1777 /tmp /var/tmp /tmp/gradio_tmp /tmp/tld /tmp/matplotlib_cache \
+    && chmod 700 ${XDG_CACHE_HOME} \
+    && mkdir -p ${APP_HOME}/.paddlex \
+    && chown user:user ${APP_HOME}/.paddlex \
+    && chmod 755 ${APP_HOME}/.paddlex \
+    && mkdir -p ${APP_HOME}/.local/share/spacy/data \
+    && chown user:user ${APP_HOME}/.local/share/spacy/data \
+    && chmod 755 ${APP_HOME}/.local/share/spacy/data \
+    && mkdir -p /usr/share/tessdata \
+    && chown user:user /usr/share/tessdata \
+    && chmod 755 /usr/share/tessdata
+
+# Fix apply user ownership to all files in the home directory
+RUN chown -R user:user /home/user
+
+# Set permissions for Python executable
+RUN chmod 755 /usr/local/bin/python
+
+# Declare volumes (NOTE: runtime mounts will override permissions — handle with care)
+VOLUME ["/tmp/matplotlib_cache"]
+VOLUME ["/tmp/gradio_tmp"]
+VOLUME ["/tmp/tld"]
+VOLUME ["/home/user/app/output"]
+VOLUME ["/home/user/app/input"]
+VOLUME ["/home/user/app/logs"]
+VOLUME ["/home/user/app/usage"]
+VOLUME ["/home/user/app/feedback"]
+VOLUME ["/home/user/app/config"]
+VOLUME ["/home/user/.paddlex"]
+VOLUME ["/home/user/.local/share/spacy/data"]
+VOLUME ["/usr/share/tessdata"]
+VOLUME ["/tmp"]
+VOLUME ["/var/tmp"]
+
+USER user
+
+EXPOSE $GRADIO_SERVER_PORT
+
+ENTRYPOINT ["/home/user/app/entrypoint.sh"]
+CMD ["python", "app.py"]

MANIFEST.in

@@ -0,0 +1,4 @@
+recursive-include doc_redaction/assets *.png
+recursive-include doc_redaction/example_data *
+recursive-include intros *.txt
+

README.md

@@ -0,0 +1,346 @@
+---
+title: Document redaction
+emoji: 📝
+colorFrom: blue
+colorTo: yellow
+sdk: docker
+app_file: app.py
+pinned: true
+license: agpl-3.0
+short_description: OCR / redact PDF documents and tabular data
+---
+# Document redaction (doc_redaction)
+
+<a href="https://pypi.org/project/doc-redaction/" target="_blank"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/doc-redaction"></a> 
+
+Redact personally identifiable information (PII) from documents (PDF, PNG, JPG), Word files (DOCX), or tabular data (XLSX/CSV/Parquet). Please see the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for a full walkthrough of all the features in the app.
+
+---
+
+## 🚀 Quick Start - Installation and first run
+
+Follow these instructions to get the document redaction application running on your local machine.
+
+### 1. Package installation
+
+#### Option 1 - Recommended: Install from source repo
+
+Clone the repository and install in editable mode:
+
+```bash
+git clone https://github.com/seanpedrick-case/doc_redaction.git
+cd doc_redaction
+pip install -e .
+```
+
+##### Install extras (Paddle or Transformers/Torch VLM)
+
+To install with PaddleOCR:
+
+```bash
+pip install -e ".[paddle]"
+```
+
+Note that the versions of both PaddleOCR and Torch installed by default are the CPU-only versions. If you want to install the equivalent GPU versions, you will need to run the following commands:
+```bash
+pip install paddlepaddle-gpu==3.2.1 --index-url https://www.paddlepaddle.org.cn/packages/stable/cu129/
+```
+
+If you want to run VLMs / LLMs with the transformers package:
+
+```bash
+pip install -e ".[vlm]"
+```
+
+
+**Note:** It is difficult to get paddlepaddle gpu working in an environment alongside torch. You may well need to reinstall the cpu version to ensure compatibility, and run paddlepaddle-gpu in a separate environment without torch installed. If you get errors related to .dll files following paddle gpu install, you may need to install the latest c++ redistributables. For Windows, you can find them [here](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170)
+
+```bash
+pip install torch==2.8.0 --index-url https://download.pytorch.org/whl/cu129
+pip install torchvision --index-url https://download.pytorch.org/whl/cu129
+```
+
+#### Option 2 - Install from PyPI
+
+Create a virtual environment (recommended) and install **doc_redaction**.
+
+```bash
+python -m venv venv
+# Windows:
+.\venv\Scripts\activate
+# macOS/Linux:
+source venv/bin/activate
+```
+
+The package is published on PyPI as **`doc-redaction`** (import name **`doc_redaction`**):
+
+```bash
+pip install doc_redaction
+```
+
+Optional extras (same as in `pyproject.toml`). For installing paddleOCR:
+
+```bash
+pip install "doc_redaction[paddle]"
+```
+
+For running VLMs / LLMs with the transformers package:
+
+```bash
+pip install "doc_redaction[vlm]"
+```
+
+For programmatic use (CLI-first API matching Gradio `api_name` routes), see **[Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html)**. The console script **`cli_redact`** is available after install.
+
+**Web UI from a PyPI install:** You *can* start the Gradio UI after `pip install doc_redaction` by running (note that the prerequisites tesseract and poppler will need to be correctly installed following step 2 below):
+
+```bash
+python -m app
+```
+
+**Important: your working directory matters.** When you run `python -m app`, the app treats your *current folder* as the “app folder”:
+
+- It will look for configuration at `config/app_config.env` *relative to the folder you run it from* (and `python -m doc_redaction.install_deps` will also write `config/app_config.env` there).
+- It may create new folders in that location (for example `config/`, `output/`, `input/`, `logs/`, `usage/`, `feedback/`, and temporary/cache folders depending on your settings).
+- The UI example files and bundled assets are packaged with the PyPI install (they live inside the installed `doc_redaction` package). If you run from a “random” directory after a PyPI install, the app can still locate its packaged examples; your working directory mainly affects where `config/`, `input/`, `output/`, logs, and temp folders are created.
+
+In practice, the **smoothest UI experience** (examples, bundled assets, docs links, predictable relative paths) is still usually via a **repository checkout** or **Docker**, but PyPI install is sufficient to launch the UI as long as you run it from a suitable working folder and have the system dependencies available (or run `python -m doc_redaction.install_deps` first).
+
+#### Option 3 - Docker installation
+
+The doc_redaction Redaction app can be installed by using the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) or Docker compose files ([llama.cpp](https://github.com/ggml-org/llama.cpp), [vLLM](https://docs.vllm.ai/en/stable/)) provided in the repo.
+
+##### With Llama.cpp / vLLM inference server
+
+The project now has Docker and Docker compose files available to pair running the Redaction app with local inference servers powered by [llama.cpp](https://github.com/ggml-org/llama.cpp), or [vLLM](https://docs.vllm.ai/en/stable/). Llama.cpp is more flexible than vLLM for low VRAM systems, as Llama.cpp will offload to cpu/system RAM automatically rather than failing as vLLM tends to do.
+
+For Llama.cpp, you can use the [docker-compose_llama.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_llama.yml) file, and for vLLM, you can use the [docker-compose_vllm.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_vllm.yml) file. To run, Docker / Docker Desktop should be installed, and then you can run the commands suggested in the top of the files to run the servers.
+
+You will need ~40 GB of disk space to run everything depending on the model chosen from the compose file. For the vLLM server, you will need 24 GB VRAM. For the Llama.cpp server, 24 GB VRAM is needed to run at full speed, but the n-gpu-layers and n-cpu-moe parameters in the Docker compose file can be adjusted to fit into your system. I would suggest that 8 GB VRAM is needed as a bare minimum for decent inference speed. See the [Unsloth guide](https://unsloth.ai/docs/models/qwen3.5) for more details on working with GGUF files for Qwen 3.5.
+
+##### Without Llama.cpp / vLLM inference server
+
+If you want a working Docker installation without GPU support, you can install from the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) in the repo. A working example of this, with the CPU version of PaddleOCR, can be found on [Hugging Face](https://huggingface.co/spaces/seanpedrickcase/document_redaction). You can adjust the INSTALL_PADDLEOCR, PADDLE_GPU_ENABLED, INSTALL_VLM, and TORCH_GPU_ENABLED config variables to adjust for PaddleOCR and Transformers packages for local VLM support. Note that GPU-enabled PaddleOCR, and GPU-enabled Transformers/Torch often don't work well together, which is one reason why a Llama.cpp/vLLM inference server Docker installation option is provided below.
+
+### 2. Install prerequisites: Tesseract and Poppler
+
+This application relies on two external tools for OCR (Tesseract) and PDF processing (Poppler). Please install them on your system before proceeding. To run the Document Redaction app successfully, these tools need to be installed and either 1. added to PATH, or 2. be in a folder that is directly referenced in the config/app_config.env file with the variables TESSERACT_FOLDER and POPPLER_FOLDER (defined [here](https://github.com/seanpedrick-case/doc_redaction/blob/main/tools/config.py) if you want to see the code). The instructions below will guide you through diffferent ways to install these dependencies.
+
+---
+
+#### Automated dependency setup (recommended)
+
+If you **don’t have admin rights** (or you just want the simplest setup), you can have the project download and configure **Tesseract** and **Poppler** into a local `redaction_deps/` folder inside the doc_redaction folder.
+
+You need the installer script available first, which means either:
+
+- **Repository checkout**: `git clone ...` and run the command from the repo root (recommended for the web UI), or
+- **PyPI install**: `pip install doc_redaction` and run from a writable folder where you want `redaction_deps/` and `config/app_config.env` to be created/updated.
+
+From the repository root (or your chosen working folder) after creating/activating your venv and installing Python requirements:
+
+```bash
+python -m doc_redaction.install_deps
+```
+
+This writes `TESSERACT_FOLDER` / `POPPLER_FOLDER` into `config/app_config.env` so the app can find the binaries without you editing your system PATH.
+
+To just check whether your machine can already see the tools:
+
+```bash
+python -m doc_redaction.install_deps --verify-only
+```
+
+#### **On Windows**
+
+If you don’t use the automated setup above, you can install the dependencies manually by downloading installers and adding the programs to your system's PATH.
+
+1.  **Install Tesseract OCR:**
+    *   Download the installer from the official Tesseract at [UB Mannheim page](https://github.com/UB-Mannheim/tesseract/wiki) (e.g., `tesseract-ocr-w64-setup-v5.X.X...exe`).
+    *   Run the installer.
+    *   **IMPORTANT:** During installation, ensure you select the option to "Add Tesseract to system PATH for all users" or a similar option. This is crucial for the application to find the Tesseract executable.
+
+
+2.  **Install Poppler:**
+    *   Download the latest Poppler binary for Windows. A common source is the [Poppler for Windows](https://github.com/oschwartz10612/poppler-windows) GitHub releases page. Download the `.zip` file (e.g., `poppler-25.07.0-win.zip`).
+    *   Extract the contents of the zip file to a permanent location on your computer, for example, `C:\Program Files\poppler\`.
+    *   You must add the `bin` folder from your Poppler installation to your system's PATH environment variable.
+        *   Search for "Edit the system environment variables" in the Windows Start Menu and open it.
+        *   Click the "Environment Variables..." button.
+        *   In the "System variables" section, find and select the `Path` variable, then click "Edit...".
+        *   Click "New" and add the full path to the `bin` directory inside your Poppler folder (e.g., `C:\Program Files\poppler\poppler-24.02.0\bin`).
+        *   Click OK on all windows to save the changes.
+
+    To verify, open a new Command Prompt and run `tesseract --version` and `pdftoppm -v`. If they both return version information, you have successfully installed the prerequisites.
+---
+
+#### **On Linux (Debian/Ubuntu)**
+
+Open your terminal and run the following command to install Tesseract and Poppler:
+
+```bash
+sudo apt-get update && sudo apt-get install -y tesseract-ocr poppler-utils
+```
+
+#### **On Linux (Fedora/CentOS/RHEL)**
+
+Open your terminal and use the `dnf` or `yum` package manager:
+
+```bash
+sudo dnf install -y tesseract poppler-utils
+```
+---
+
+### 3. Run the Application
+
+With all dependencies installed, you can now start the Gradio application GUI. For a guide on how to use this, please go [here](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html).
+
+```bash
+python app.py
+```
+
+After running the command, the application will start, and you will see a local URL in your terminal (usually `http://127.0.0.1:7860`).
+
+Open this URL in your web browser to use the document redaction tool
+
+#### Command line interface
+
+For example CLI commands, please refer to [this guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html#command-line-interface-cli) or the examples in [cli_redact.py](https://github.com/seanpedrick-case/doc_redaction/blob/main/cli_redact.py#L321)
+
+If you installed from **PyPI**, use the installed console script:
+
+```bash
+cli_redact --help
+```
+
+From a **repository checkout**, you can also run:
+
+```bash
+python cli_redact.py --help
+```
+
+#### Python package commands
+
+For Python examples in using the Python package, please see [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+
+---
+
+
+### 4. ⚙️ Configuration (Optional)
+
+You can customise the application's behavior by creating a configuration file. This allows you to change settings without modifying the source code, such as enabling AWS features, changing logging behavior, or pointing to local Tesseract/Poppler installations. A full overview of all the potential settings you can modify in the app_config.env file can be seen in tools/config.py, with explanation on the documentation website for [the github repo](https://seanpedrick-case.github.io/doc_redaction/)
+
+To get started:
+1.  Locate the `example_config.env` file in the root of the project.
+2.  Create a new file named `app_config.env` inside the `config/` directory (i.e., `config/app_config.env`).
+3.  Copy the contents from `example_config.env` into your new `config/app_config.env` file.
+4.  Modify the values in `config/app_config.env` to suit your needs. The application will automatically load these settings on startup.
+
+If you do not create this file, the application will run with default settings.
+
+#### Configuration Breakdown
+
+Here is an overview of the most important settings, separated by whether they are for local use or require AWS.
+
+---
+
+#### **Local & General Settings (No AWS Required)**
+
+These settings are useful for all users, regardless of whether you are using AWS.
+
+*   `TESSERACT_FOLDER` / `POPPLER_FOLDER`
+    *   Use these if you installed Tesseract or Poppler to a custom location on **Windows** and did not add them to the system PATH.
+    *   Provide the path to the respective installation folders (for Poppler, point to the `bin` sub-directory).
+    *   **Examples:** `POPPLER_FOLDER=C:/Program Files/poppler-24.02.0/bin/` `TESSERACT_FOLDER=tesseract/`
+
+*   `TESSERACT_DATA_FOLDER`
+    *   If Tesseract runs but you see an error like `Error opening data file ./eng.traineddata` or `Tesseract couldn't load any languages`, this is usually because it can't find the `tessdata/` language files.
+    *   Set this to the folder that contains `eng.traineddata` (typically a `tessdata` directory).
+    *   **Examples (Windows):** `TESSERACT_DATA_FOLDER=C:/Program Files/Tesseract-OCR/tessdata`
+
+*   `SHOW_LANGUAGE_SELECTION=True`
+    *   Set to `True` to display a language selection dropdown in the UI for OCR processing.
+
+*   `DEFAULT_LOCAL_OCR_MODEL=tesseract`"
+    *   Choose the backend for local OCR. Options are `tesseract`, `paddle`, or `hybrid`. "Tesseract" is the default, and is recommended. "hybrid-paddle" is a combination of the two - first pass through the redactions will be done with Tesseract, and then a second pass will be done with PaddleOCR on words with low confidence. "paddle" will only return whole line text extraction, and so will only work for OCR, not redaction. 
+
+*   `SESSION_OUTPUT_FOLDER=False`
+    *   If `True`, redacted files will be saved in unique subfolders within the `output/` directory for each session.
+
+*   `DISPLAY_FILE_NAMES_IN_LOGS=False`
+    *   For privacy, file names are not recorded in usage logs by default. Set to `True` to include them.
+
+---
+
+#### **AWS-Specific Settings**
+
+These settings are only relevant if you intend to use AWS services like Textract for OCR and Comprehend for PII detection.
+
+*   `RUN_AWS_FUNCTIONS=True`
+    *   **This is the master switch.** You must set this to `True` to enable any AWS functionality. If it is `False`, all other AWS settings will be ignored.
+
+*   **UI Options:**
+    *   `SHOW_AWS_TEXT_EXTRACTION_OPTIONS=True`: Adds "AWS Textract" as an option in the text extraction dropdown.
+    *   `SHOW_AWS_PII_DETECTION_OPTIONS=True`: Adds "AWS Comprehend" as an option in the PII detection dropdown.
+
+*   **Core AWS Configuration:**
+    *   `AWS_REGION=example-region`: Set your AWS region (e.g., `us-east-1`).
+    *   `DOCUMENT_REDACTION_BUCKET=example-bucket`: The name of the S3 bucket the application will use for temporary file storage and processing.
+
+*   **AWS Logging:**
+    *   `SAVE_LOGS_TO_DYNAMODB=True`: If enabled, usage and feedback logs will be saved to DynamoDB tables.
+    *   `ACCESS_LOG_DYNAMODB_TABLE_NAME`, `USAGE_LOG_DYNAMODB_TABLE_NAME`, etc.: Specify the names of your DynamoDB tables for logging.
+
+*   **Advanced AWS Textract Features:**
+    *   `SHOW_WHOLE_DOCUMENT_TEXTRACT_CALL_OPTIONS=True`: Enables UI components for large-scale, asynchronous document processing via Textract.
+    *   `TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET=example-bucket-output`: A separate S3 bucket for the final output of asynchronous Textract jobs.
+    *   `LOAD_PREVIOUS_TEXTRACT_JOBS_S3=True`: If enabled, the app will try to load the status of previously submitted asynchronous jobs from S3.
+
+*   **Cost Tracking (for internal accounting):**
+    *   `SHOW_COSTS=True`: Displays an estimated cost for AWS operations. Can be enabled even if AWS functions are off.
+    *   `GET_COST_CODES=True`: Enables a dropdown for users to select a cost code before running a job.
+    *   `COST_CODES_PATH=config/cost_codes.csv`: The local path to a CSV file containing your cost codes.
+    *   `ENFORCE_COST_CODES=True`: Makes selecting a cost code mandatory before starting a redaction.
+
+Now you have the app installed, please refer to the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for more information on how to use it for basic and advanced redaction.
+
+## For agents (API quickstart)
+
+If you are an LLM/agent interacting with this app over HTTP (e.g. Hugging Face Spaces), **do not guess inputs** from the UI. Use the Gradio schema as the source of truth:
+
+- **Discover schema**: `GET /gradio_api/info`
+- **Upload files**: `POST /gradio_api/upload` (multipart field `files`) → returns server-internal paths like `/tmp/gradio_tmp/...`
+- **Call**: `POST /gradio_api/call/{api_name}` with body `{"data":[...]}` (argument order must match `/gradio_api/info`)
+- **Poll**: `GET /gradio_api/call/{api_name}/{event_id}` until complete
+- **Download outputs**: `GET /gradio_api/file={path}` (note: some deployments return 403 without session cookies)
+
+### Choose the correct route (prefer short `gr.api` endpoints)
+
+Fetch `/gradio_api/info` and then prefer the simplest route that exists:
+
+- **Apply edited review CSV to a PDF**: `/review_apply`
+- **Redact a PDF/image document**: `/doc_redact`
+- **Summarise a PDF**: `/pdf_summarise`
+- **Redact tabular files (CSV/XLSX/Parquet/DOCX)**: `/tabular_redact`
+
+If those endpoints are not present in your deployment, fall back to the long UI-chained routes (`/apply_review_redactions`, `/redact_data`, etc.) and build `data[]` strictly from `/gradio_api/info`.
+
+### Common gotchas
+
+- **Arity errors** (`needed: N, got: M`) mean you called a session-heavy UI handler with the wrong `data[]`. Prefer the short endpoints above.
+- **`handle_file()` gotcha** (for `gradio_client` users): do **not** wrap server-internal upload paths (e.g. `/tmp/gradio_tmp/...`) with `handle_file()`. Pass them as plain strings.
+- **Container-only outputs**: outputs may be written to container paths (e.g. `/home/user/app/output/`). Plan to download via `file=...` or use a mounted output directory in Docker.
+
+### Optional: MCP server
+
+If you want external agents to call this app reliably without re-implementing Gradio upload/call/poll/download details, consider an **MCP server** that wraps the main tasks (`redact_document`, `apply_review_redactions`, `redact_tabular`, `summarise_document`) behind a small tool interface. See the [relevant documentation](https://github.com/seanpedrick-case/doc_redaction/blob/main/mcp_doc_redaction/README.md).
+
+**Use as a library:** After installing from [PyPI](https://pypi.org/project/doc-redaction/) (`pip install doc_redaction`), you can call the same workflows as the Gradio `api_name` routes from Python. See the documentation: [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+    
+To extract text from documents, the 'Local' options are PikePDF for PDFs with selectable text, and OCR with Tesseract. Use AWS Textract to extract more complex elements e.g. handwriting, signatures, or unclear text. PaddleOCR and VLM support is also provided (see the installation instructions below). 
+
+For PII identification, 'Local' (based on spaCy) gives good results if you are looking for common names or terms, or a custom list of terms to redact (see Redaction settings).  AWS Comprehend gives better results at a small cost.
+
+Additional options on the 'Redaction settings' include, the type of information to redact (e.g. people, places), custom terms to include/ exclude from redaction, fuzzy matching, language settings, and whole page redaction. After redaction is complete, you can view and modify suggested redactions on the 'Review redactions' tab to quickly create a final redacted document.
+
+NOTE: The app is not 100% accurate, and it will miss some personal information. It is essential that all outputs are reviewed **by a human** before using the final outputs.

README_PYPI.md

@@ -0,0 +1,330 @@
+# Document redaction (doc_redaction)
+
+<a href="https://pypi.org/project/doc-redaction/" target="_blank"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/doc-redaction"></a> 
+
+Redact personally identifiable information (PII) from documents (PDF, PNG, JPG), Word files (DOCX), or tabular data (XLSX/CSV/Parquet). Please see the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for a full walkthrough of all the features in the app.
+
+---
+
+## 🚀 Quick Start - Installation and first run
+
+Follow these instructions to get the document redaction application running on your local machine.
+
+### 1. Package installation
+
+#### Option 1 - Recommended: Install from source repo
+
+Clone the repository and install in editable mode:
+
+```bash
+git clone https://github.com/seanpedrick-case/doc_redaction.git
+cd doc_redaction
+pip install -e .
+```
+
+##### Install extras (Paddle or Transformers/Torch VLM)
+
+To install with PaddleOCR:
+
+```bash
+pip install -e ".[paddle]"
+```
+
+Note that the versions of both PaddleOCR and Torch installed by default are the CPU-only versions. If you want to install the equivalent GPU versions, you will need to run the following commands:
+```bash
+pip install paddlepaddle-gpu==3.2.1 --index-url https://www.paddlepaddle.org.cn/packages/stable/cu129/
+```
+
+If you want to run VLMs / LLMs with the transformers package:
+
+```bash
+pip install -e ".[vlm]"
+```
+
+
+**Note:** It is difficult to get paddlepaddle gpu working in an environment alongside torch. You may well need to reinstall the cpu version to ensure compatibility, and run paddlepaddle-gpu in a separate environment without torch installed. If you get errors related to .dll files following paddle gpu install, you may need to install the latest c++ redistributables. For Windows, you can find them [here](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170)
+
+```bash
+pip install torch==2.8.0 --index-url https://download.pytorch.org/whl/cu129
+pip install torchvision --index-url https://download.pytorch.org/whl/cu129
+```
+
+#### Option 2 - Install from PyPI
+
+Create a virtual environment (recommended) and install **doc_redaction**.
+
+```bash
+python -m venv venv
+# Windows:
+.\venv\Scripts\activate
+# macOS/Linux:
+source venv/bin/activate
+```
+
+The package is published on PyPI as **`doc-redaction`** (import name **`doc_redaction`**):
+
+```bash
+pip install doc_redaction
+```
+
+Optional extras (same as in `pyproject.toml`). For installing paddleOCR:
+
+```bash
+pip install "doc_redaction[paddle]"
+```
+
+For running VLMs / LLMs with the transformers package:
+
+```bash
+pip install "doc_redaction[vlm]"
+```
+
+For programmatic use (CLI-first API matching Gradio `api_name` routes), see **[Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html)**. The console script **`cli_redact`** is available after install.
+
+**Web UI from a PyPI install:** You *can* start the Gradio UI after `pip install doc_redaction` by running (note that the prerequisites tesseract and poppler will need to be correctly installed following step 2 below):
+
+```bash
+python -m app
+```
+
+**Important: your working directory matters.** When you run `python -m app`, the app treats your *current folder* as the “app folder”:
+
+- It will look for configuration at `config/app_config.env` *relative to the folder you run it from* (and `python -m doc_redaction.install_deps` will also write `config/app_config.env` there).
+- It may create new folders in that location (for example `config/`, `output/`, `input/`, `logs/`, `usage/`, `feedback/`, and temporary/cache folders depending on your settings).
+- The UI example files and bundled assets are packaged with the PyPI install (they live inside the installed `doc_redaction` package). If you run from a “random” directory after a PyPI install, the app can still locate its packaged examples; your working directory mainly affects where `config/`, `input/`, `output/`, logs, and temp folders are created.
+
+In practice, the **smoothest UI experience** (examples, bundled assets, docs links, predictable relative paths) is still usually via a **repository checkout** or **Docker**, but PyPI install is sufficient to launch the UI as long as you run it from a suitable working folder and have the system dependencies available (or run `python -m doc_redaction.install_deps` first).
+
+#### Option 3 - Docker installation
+
+The doc_redaction Redaction app can be installed by using the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) or Docker compose files ([llama.cpp](https://github.com/ggml-org/llama.cpp), [vLLM](https://docs.vllm.ai/en/stable/)) provided in the repo.
+
+##### With Llama.cpp / vLLM inference server
+
+The project now has Docker and Docker compose files available to pair running the Redaction app with local inference servers powered by [llama.cpp](https://github.com/ggml-org/llama.cpp), or [vLLM](https://docs.vllm.ai/en/stable/). Llama.cpp is more flexible than vLLM for low VRAM systems, as Llama.cpp will offload to cpu/system RAM automatically rather than failing as vLLM tends to do.
+
+For Llama.cpp, you can use the [docker-compose_llama.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_llama.yml) file, and for vLLM, you can use the [docker-compose_vllm.yml](https://github.com/seanpedrick-case/doc_redaction/blob/main/docker-compose_vllm.yml) file. To run, Docker / Docker Desktop should be installed, and then you can run the commands suggested in the top of the files to run the servers.
+
+You will need ~40 GB of disk space to run everything depending on the model chosen from the compose file. For the vLLM server, you will need 24 GB VRAM. For the Llama.cpp server, 24 GB VRAM is needed to run at full speed, but the n-gpu-layers and n-cpu-moe parameters in the Docker compose file can be adjusted to fit into your system. I would suggest that 8 GB VRAM is needed as a bare minimum for decent inference speed. See the [Unsloth guide](https://unsloth.ai/docs/models/qwen3.5) for more details on working with GGUF files for Qwen 3.5.
+
+##### Without Llama.cpp / vLLM inference server
+
+If you want a working Docker installation without GPU support, you can install from the [Dockerfile](https://github.com/seanpedrick-case/doc_redaction/blob/main/Dockerfile) in the repo. A working example of this, with the CPU version of PaddleOCR, can be found on [Hugging Face](https://huggingface.co/spaces/seanpedrickcase/document_redaction). You can adjust the INSTALL_PADDLEOCR, PADDLE_GPU_ENABLED, INSTALL_VLM, and TORCH_GPU_ENABLED config variables to adjust for PaddleOCR and Transformers packages for local VLM support. Note that GPU-enabled PaddleOCR, and GPU-enabled Transformers/Torch often don't work well together, which is one reason why a Llama.cpp/vLLM inference server Docker installation option is provided below.
+
+### 2. Install prerequisites: Tesseract and Poppler
+
+This application relies on two external tools for OCR (Tesseract) and PDF processing (Poppler). Please install them on your system before proceeding.
+
+---
+
+#### Automated dependency setup (recommended)
+
+If you **don’t have admin rights** (or you just want the simplest setup), you can have the project download and configure **Tesseract** and **Poppler** into a local `redaction_deps/` folder inside the doc_redaction folder.
+
+You need the installer script available first, which means either:
+
+- **Repository checkout**: `git clone ...` and run the command from the repo root (recommended for the web UI), or
+- **PyPI install**: `pip install doc_redaction` and run from a writable folder where you want `redaction_deps/` and `config/app_config.env` to be created/updated.
+
+From the repository root (or your chosen working folder) after creating/activating your venv and installing Python requirements:
+
+```bash
+python -m doc_redaction.install_deps
+```
+
+This writes `TESSERACT_FOLDER` / `POPPLER_FOLDER` into `config/app_config.env` so the app can find the binaries without you editing your system PATH.
+
+To just check whether your machine can already see the tools:
+
+```bash
+python -m doc_redaction.install_deps --verify-only
+```
+
+#### **On Windows**
+
+If you don’t use the automated setup above, you can install the dependencies manually by downloading installers and adding the programs to your system's PATH.
+
+1.  **Install Tesseract OCR:**
+    *   Download the installer from the official Tesseract at [UB Mannheim page](https://github.com/UB-Mannheim/tesseract/wiki) (e.g., `tesseract-ocr-w64-setup-v5.X.X...exe`).
+    *   Run the installer.
+    *   **IMPORTANT:** During installation, ensure you select the option to "Add Tesseract to system PATH for all users" or a similar option. This is crucial for the application to find the Tesseract executable.
+
+
+2.  **Install Poppler:**
+    *   Download the latest Poppler binary for Windows. A common source is the [Poppler for Windows](https://github.com/oschwartz10612/poppler-windows) GitHub releases page. Download the `.zip` file (e.g., `poppler-25.07.0-win.zip`).
+    *   Extract the contents of the zip file to a permanent location on your computer, for example, `C:\Program Files\poppler\`.
+    *   You must add the `bin` folder from your Poppler installation to your system's PATH environment variable.
+        *   Search for "Edit the system environment variables" in the Windows Start Menu and open it.
+        *   Click the "Environment Variables..." button.
+        *   In the "System variables" section, find and select the `Path` variable, then click "Edit...".
+        *   Click "New" and add the full path to the `bin` directory inside your Poppler folder (e.g., `C:\Program Files\poppler\poppler-24.02.0\bin`).
+        *   Click OK on all windows to save the changes.
+
+    To verify, open a new Command Prompt and run `tesseract --version` and `pdftoppm -v`. If they both return version information, you have successfully installed the prerequisites.
+---
+
+#### **On Linux (Debian/Ubuntu)**
+
+Open your terminal and run the following command to install Tesseract and Poppler:
+
+```bash
+sudo apt-get update && sudo apt-get install -y tesseract-ocr poppler-utils
+```
+
+#### **On Linux (Fedora/CentOS/RHEL)**
+
+Open your terminal and use the `dnf` or `yum` package manager:
+
+```bash
+sudo dnf install -y tesseract poppler-utils
+```
+---
+
+### 3. Run the Application
+
+With all dependencies installed, you can now start the Gradio application GUI. For a guide on how to use this, please go [here](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html).
+
+```bash
+python app.py
+```
+
+After running the command, the application will start, and you will see a local URL in your terminal (usually `http://127.0.0.1:7860`).
+
+Open this URL in your web browser to use the document redaction tool
+
+#### Command line interface
+
+For example CLI commands, please refer to [this guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html#command-line-interface-cli) or the examples in [cli_redact.py](https://github.com/seanpedrick-case/doc_redaction/blob/main/cli_redact.py#L321)
+
+If you installed from **PyPI**, use the installed console script:
+
+```bash
+cli_redact --help
+```
+
+From a **repository checkout**, you can also run:
+
+```bash
+python cli_redact.py --help
+```
+
+#### Python package commands
+
+For Python examples in using the Python package, please see [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+
+---
+
+
+### 4. ⚙️ Configuration (Optional)
+
+You can customise the application's behavior by creating a configuration file. This allows you to change settings without modifying the source code, such as enabling AWS features, changing logging behavior, or pointing to local Tesseract/Poppler installations. A full overview of all the potential settings you can modify in the app_config.env file can be seen in tools/config.py, with explanation on the documentation website for [the github repo](https://seanpedrick-case.github.io/doc_redaction/)
+
+To get started:
+1.  Locate the `example_config.env` file in the root of the project.
+2.  Create a new file named `app_config.env` inside the `config/` directory (i.e., `config/app_config.env`).
+3.  Copy the contents from `example_config.env` into your new `config/app_config.env` file.
+4.  Modify the values in `config/app_config.env` to suit your needs. The application will automatically load these settings on startup.
+
+If you do not create this file, the application will run with default settings.
+
+#### Configuration Breakdown
+
+Here is an overview of the most important settings, separated by whether they are for local use or require AWS.
+
+---
+
+#### **Local & General Settings (No AWS Required)**
+
+These settings are useful for all users, regardless of whether you are using AWS.
+
+*   `TESSERACT_FOLDER` / `POPPLER_FOLDER`
+    *   Use these if you installed Tesseract or Poppler to a custom location on **Windows** and did not add them to the system PATH.
+    *   Provide the path to the respective installation folders (for Poppler, point to the `bin` sub-directory).
+    *   **Examples:** `POPPLER_FOLDER=C:/Program Files/poppler-24.02.0/bin/` `TESSERACT_FOLDER=tesseract/`
+
+*   `SHOW_LANGUAGE_SELECTION=True`
+    *   Set to `True` to display a language selection dropdown in the UI for OCR processing.
+
+*   `DEFAULT_LOCAL_OCR_MODEL=tesseract`"
+    *   Choose the backend for local OCR. Options are `tesseract`, `paddle`, or `hybrid`. "Tesseract" is the default, and is recommended. "hybrid-paddle" is a combination of the two - first pass through the redactions will be done with Tesseract, and then a second pass will be done with PaddleOCR on words with low confidence. "paddle" will only return whole line text extraction, and so will only work for OCR, not redaction. 
+
+*   `SESSION_OUTPUT_FOLDER=False`
+    *   If `True`, redacted files will be saved in unique subfolders within the `output/` directory for each session.
+
+*   `DISPLAY_FILE_NAMES_IN_LOGS=False`
+    *   For privacy, file names are not recorded in usage logs by default. Set to `True` to include them.
+
+---
+
+#### **AWS-Specific Settings**
+
+These settings are only relevant if you intend to use AWS services like Textract for OCR and Comprehend for PII detection.
+
+*   `RUN_AWS_FUNCTIONS=True`
+    *   **This is the master switch.** You must set this to `True` to enable any AWS functionality. If it is `False`, all other AWS settings will be ignored.
+
+*   **UI Options:**
+    *   `SHOW_AWS_TEXT_EXTRACTION_OPTIONS=True`: Adds "AWS Textract" as an option in the text extraction dropdown.
+    *   `SHOW_AWS_PII_DETECTION_OPTIONS=True`: Adds "AWS Comprehend" as an option in the PII detection dropdown.
+
+*   **Core AWS Configuration:**
+    *   `AWS_REGION=example-region`: Set your AWS region (e.g., `us-east-1`).
+    *   `DOCUMENT_REDACTION_BUCKET=example-bucket`: The name of the S3 bucket the application will use for temporary file storage and processing.
+
+*   **AWS Logging:**
+    *   `SAVE_LOGS_TO_DYNAMODB=True`: If enabled, usage and feedback logs will be saved to DynamoDB tables.
+    *   `ACCESS_LOG_DYNAMODB_TABLE_NAME`, `USAGE_LOG_DYNAMODB_TABLE_NAME`, etc.: Specify the names of your DynamoDB tables for logging.
+
+*   **Advanced AWS Textract Features:**
+    *   `SHOW_WHOLE_DOCUMENT_TEXTRACT_CALL_OPTIONS=True`: Enables UI components for large-scale, asynchronous document processing via Textract.
+    *   `TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET=example-bucket-output`: A separate S3 bucket for the final output of asynchronous Textract jobs.
+    *   `LOAD_PREVIOUS_TEXTRACT_JOBS_S3=True`: If enabled, the app will try to load the status of previously submitted asynchronous jobs from S3.
+
+*   **Cost Tracking (for internal accounting):**
+    *   `SHOW_COSTS=True`: Displays an estimated cost for AWS operations. Can be enabled even if AWS functions are off.
+    *   `GET_COST_CODES=True`: Enables a dropdown for users to select a cost code before running a job.
+    *   `COST_CODES_PATH=config/cost_codes.csv`: The local path to a CSV file containing your cost codes.
+    *   `ENFORCE_COST_CODES=True`: Makes selecting a cost code mandatory before starting a redaction.
+
+Now you have the app installed, please refer to the [User Guide](https://seanpedrick-case.github.io/doc_redaction/src/user_guide.html) for more information on how to use it for basic and advanced redaction.
+
+## For agents (API quickstart)
+
+If you are an LLM/agent interacting with this app over HTTP (e.g. Hugging Face Spaces), **do not guess inputs** from the UI. Use the Gradio schema as the source of truth:
+
+- **Discover schema**: `GET /gradio_api/info`
+- **Upload files**: `POST /gradio_api/upload` (multipart field `files`) → returns server-internal paths like `/tmp/gradio_tmp/...`
+- **Call**: `POST /gradio_api/call/{api_name}` with body `{"data":[...]}` (argument order must match `/gradio_api/info`)
+- **Poll**: `GET /gradio_api/call/{api_name}/{event_id}` until complete
+- **Download outputs**: `GET /gradio_api/file={path}` (note: some deployments return 403 without session cookies)
+
+### Choose the correct route (prefer short `gr.api` endpoints)
+
+Fetch `/gradio_api/info` and then prefer the simplest route that exists:
+
+- **Apply edited review CSV to a PDF**: `/review_apply`
+- **Redact a PDF/image document**: `/doc_redact`
+- **Summarise a PDF**: `/pdf_summarise`
+- **Redact tabular files (CSV/XLSX/Parquet/DOCX)**: `/tabular_redact`
+
+If those endpoints are not present in your deployment, fall back to the long UI-chained routes (`/apply_review_redactions`, `/redact_data`, etc.) and build `data[]` strictly from `/gradio_api/info`.
+
+### Common gotchas
+
+- **Arity errors** (`needed: N, got: M`) mean you called a session-heavy UI handler with the wrong `data[]`. Prefer the short endpoints above.
+- **`handle_file()` gotcha** (for `gradio_client` users): do **not** wrap server-internal upload paths (e.g. `/tmp/gradio_tmp/...`) with `handle_file()`. Pass them as plain strings.
+- **Container-only outputs**: outputs may be written to container paths (e.g. `/home/user/app/output/`). Plan to download via `file=...` or use a mounted output directory in Docker.
+
+### Optional: MCP server
+
+If you want external agents to call this app reliably without re-implementing Gradio upload/call/poll/download details, consider an **MCP server** that wraps the main tasks (`redact_document`, `apply_review_redactions`, `redact_tabular`, `summarise_document`) behind a small tool interface. See the [relevant documentation](https://github.com/seanpedrick-case/doc_redaction/blob/main/mcp_doc_redaction/README.md).
+
+**Use as a library:** After installing from [PyPI](https://pypi.org/project/doc-redaction/) (`pip install doc_redaction`), you can call the same workflows as the Gradio `api_name` routes from Python. See the documentation: [Python Package usage (Python)](https://seanpedrick-case.github.io/doc_redaction/src/python_package_usage.html).
+    
+To extract text from documents, the 'Local' options are PikePDF for PDFs with selectable text, and OCR with Tesseract. Use AWS Textract to extract more complex elements e.g. handwriting, signatures, or unclear text. PaddleOCR and VLM support is also provided (see the installation instructions below). 
+
+For PII identification, 'Local' (based on spaCy) gives good results if you are looking for common names or terms, or a custom list of terms to redact (see Redaction settings).  AWS Comprehend gives better results at a small cost.
+
+Additional options on the 'Redaction settings' include, the type of information to redact (e.g. people, places), custom terms to include/ exclude from redaction, fuzzy matching, language settings, and whole page redaction. After redaction is complete, you can view and modify suggested redactions on the 'Review redactions' tab to quickly create a final redacted document.
+
+NOTE: The app is not 100% accurate, and it will miss some personal information. It is essential that all outputs are reviewed **by a human** before using the final outputs.

agent_routes.py

@@ -0,0 +1,1020 @@
+"""
+FastAPI routes for programmatic / agent callers.
+
+HTTP paths align with Gradio ``api_name`` values in app.py. See GET /agent/operations
+for the full map. Uses cli_redact.main(direct_mode_args=...) where a CLI task exists.
+"""
+
+from __future__ import annotations
+
+import io
+import os
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, Depends, Header, HTTPException
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field, field_validator
+
+from tools.config import (
+    AWS_LLM_PII_OPTION,
+    AWS_PII_OPTION,
+    INFERENCE_SERVER_PII_OPTION,
+    INPUT_FOLDER,
+    LOCAL_OCR_MODEL_OPTIONS,
+    LOCAL_PII_OPTION,
+    LOCAL_TRANSFORMERS_LLM_PII_OPTION,
+    OUTPUT_FOLDER,
+)
+from tools.secure_path_utils import validate_path_safety
+
+router = APIRouter(tags=["Agent"])
+
+REPO_ROOT = Path(__file__).resolve().parent
+_MAX_INSTRUCTION_LEN = 16_000
+
+# NOTE: Paths from request bodies are untrusted. Avoid Path.resolve() on untrusted
+# input (CodeQL py/path-injection); instead normalize via os.path and enforce
+# containment under trusted roots.
+
+# Mirrors app.py api_name values (Gradio).
+GRADIO_API_NAMES: tuple[str, ...] = (
+    "redact_document",
+    "load_and_prepare_documents_or_data",
+    "apply_review_redactions",
+    "review_apply",
+    "pdf_summarise",
+    "tabular_redact",
+    "word_level_ocr_text_search",
+    "redact_data",
+    "find_duplicate_pages",
+    "find_duplicate_tabular",
+    "summarise_document",
+    "combine_review_csvs",
+    "combine_review_pdfs",
+    "export_review_redaction_overlay",
+    "export_review_page_ocr_visualisation",
+)
+
+
+def _allowed_path_roots() -> list[Path]:
+    # Return roots without resolving. These are trusted config values, but avoiding
+    # Path.resolve() keeps CodeQL happy and matches our "no resolve on untrusted"
+    # approach elsewhere.
+    roots = [REPO_ROOT]
+    for folder in (INPUT_FOLDER, OUTPUT_FOLDER):
+        if folder:
+            roots.append(Path(str(folder)))
+    return roots
+
+
+def _sanitize_untrusted_path_input(path_str: str) -> str:
+    """Basic raw-input validation before any path normalization."""
+    if not isinstance(path_str, str):
+        raise HTTPException(status_code=400, detail="Path must be a string.")
+    cleaned = path_str.strip()
+    if not cleaned:
+        raise HTTPException(status_code=400, detail="Path must not be empty.")
+    if "\x00" in cleaned:
+        raise HTTPException(status_code=400, detail="Path contains invalid null byte.")
+    return cleaned
+
+
+def _normalize_untrusted_path_to_abs(path_str: str) -> str:
+    """
+    Expand ~, then normalize to an absolute path.
+
+    Relative paths are interpreted relative to REPO_ROOT (matching prior behaviour).
+    """
+    safe_input = _sanitize_untrusted_path_input(path_str)
+    expanded = os.path.expanduser(safe_input)
+    if os.path.isabs(expanded):
+        return os.path.normpath(os.path.abspath(expanded))
+    return os.path.normpath(os.path.abspath(os.path.join(str(REPO_ROOT), expanded)))
+
+
+def _must_be_under_allowed_roots(candidate_abs: str, original: str) -> None:
+    """Enforce candidate is contained under repo, INPUT_FOLDER, or OUTPUT_FOLDER."""
+    candidate_real = os.path.realpath(str(candidate_abs))
+    allowed_roots = [
+        os.path.realpath(os.path.abspath(str(p))) for p in _allowed_path_roots()
+    ]
+    for root in allowed_roots:
+        try:
+            common = os.path.commonpath([candidate_real, root])
+        except ValueError:
+            # Different drive on Windows or invalid path mix
+            continue
+        if common == root:
+            return
+    raise HTTPException(
+        status_code=403,
+        detail="Path must be under the app repo, INPUT_FOLDER, or OUTPUT_FOLDER",
+    )
+
+
+def _path_must_be_allowed_file(path_str: str) -> str:
+    """Resolve path, ensure it is under an allowed root and exists as a file."""
+    candidate_abs = _normalize_untrusted_path_to_abs(path_str)
+    candidate_real = os.path.realpath(candidate_abs)
+
+    # Validate both "safe path" patterns and containment under trusted roots.
+    _must_be_under_allowed_roots(candidate_real, path_str)
+    ok = any(
+        validate_path_safety(candidate_real, base_path=str(root))
+        for root in _allowed_path_roots()
+    )
+    if not ok:
+        raise HTTPException(status_code=400, detail=f"Unsafe path rejected: {path_str}")
+    try:
+        candidate_path = Path(candidate_real)
+        if not candidate_path.is_file():
+            raise HTTPException(
+                status_code=400, detail=f"Not a file or missing: {candidate_real}"
+            )
+    except OSError:
+        raise HTTPException(
+            status_code=400, detail=f"Not a file or missing: {candidate_real}"
+        )
+    return candidate_real
+
+
+def _path_must_be_allowed_directory(path_str: str, *, must_exist: bool = True) -> str:
+    """
+    Normalize and validate a directory path under allowed roots.
+
+    By default the directory must already exist; callers can opt out (e.g. output_dir
+    that will be created later by the CLI).
+    """
+    candidate_abs = _normalize_untrusted_path_to_abs(path_str)
+    candidate_real = os.path.realpath(candidate_abs)
+
+    _must_be_under_allowed_roots(candidate_real, path_str)
+    ok = any(
+        validate_path_safety(candidate_real, base_path=str(root))
+        for root in _allowed_path_roots()
+    )
+    if not ok:
+        raise HTTPException(status_code=400, detail=f"Unsafe path rejected: {path_str}")
+    if must_exist:
+        try:
+            if not Path(candidate_real).is_dir():
+                raise HTTPException(
+                    status_code=400, detail=f"Not a directory: {candidate_real}"
+                )
+        except OSError:
+            raise HTTPException(
+                status_code=400, detail=f"Not a directory: {candidate_real}"
+            )
+    return candidate_real
+
+
+def _optional_agent_api_key(x_agent_api_key: Optional[str] = Header(None)) -> None:
+    expected = os.environ.get("AGENT_API_KEY", "").strip()
+    if not expected:
+        return
+    if not x_agent_api_key or x_agent_api_key.strip() != expected:
+        raise HTTPException(
+            status_code=401,
+            detail="Set header X-Agent-API-Key to match AGENT_API_KEY environment variable",
+        )
+
+
+class AgentRedactDocumentRequest(BaseModel):
+    """Parity with Gradio api_name ``redact_document``."""
+
+    input_files: list[str] = Field(
+        ...,
+        min_length=1,
+        description="Paths to input files (PDF, images, or tabular/Word for anonymisation)",
+    )
+    instruction: Optional[str] = Field(
+        None,
+        description="Optional instructions for LLM-based PII detection (custom_llm_instructions)",
+    )
+    output_dir: Optional[str] = None
+    input_dir: Optional[str] = None
+    ocr_method: Optional[str] = Field(
+        None,
+        description=(
+            "High-level OCR/text mode. Accepted values: 'Local OCR', "
+            "'AWS Textract', 'Local text'. To choose a specific local OCR engine "
+            "(e.g. paddle/tesseract/vlm), set "
+            "overrides.chosen_local_ocr_model."
+        ),
+    )
+    pii_detector: Optional[str] = Field(
+        None,
+        description=(
+            "PII detection method. Recommended configured labels: "
+            f"'{LOCAL_PII_OPTION}', '{AWS_PII_OPTION}', '{AWS_LLM_PII_OPTION}', "
+            f"'{INFERENCE_SERVER_PII_OPTION}', '{LOCAL_TRANSFORMERS_LLM_PII_OPTION}', "
+            "'None'."
+        ),
+    )
+    overrides: Optional[dict[str, Any]] = Field(
+        None,
+        description=(
+            "Optional CLI flag overrides; keys must match argparse destination names. "
+            "For local OCR model selection, set 'chosen_local_ocr_model' "
+            f"(allowed models depend on deployment; configured options: {LOCAL_OCR_MODEL_OPTIONS})."
+        ),
+    )
+
+    model_config = {
+        "json_schema_extra": {
+            "examples": [
+                {
+                    "input_files": [
+                        "example_data/example_of_emails_sent_to_a_professor_before_applying.pdf"
+                    ],
+                    "instruction": "Do not redact the university name.",
+                    "ocr_method": "Local OCR",
+                    "pii_detector": LOCAL_PII_OPTION,
+                    "overrides": {"chosen_local_ocr_model": "paddle"},
+                }
+            ]
+        }
+    }
+
+    @field_validator("instruction")
+    @classmethod
+    def _cap_instruction(cls, v: Optional[str]) -> Optional[str]:
+        if v is None:
+            return v
+        if len(v) > _MAX_INSTRUCTION_LEN:
+            raise ValueError(f"instruction exceeds {_MAX_INSTRUCTION_LEN} characters")
+        return v
+
+
+class AgentRedactDataRequest(AgentRedactDocumentRequest):
+    """Parity with Gradio api_name ``redact_data``; same CLI task as redact_document."""
+
+
+class AgentTaskResponse(BaseModel):
+    status: str
+    gradio_api_name: str
+    task: str
+    output_dir: str
+    input_dir: str
+    message: str
+    log_excerpt: Optional[str] = None
+    output_paths: Optional[list[str]] = None
+
+
+def _merge_redact_direct_mode(body: AgentRedactDocumentRequest) -> dict[str, Any]:
+    from cli_redact import get_cli_default_args_dict
+
+    merged: dict[str, Any] = get_cli_default_args_dict()
+    merged["task"] = "redact"
+    merged["input_file"] = [_path_must_be_allowed_file(p) for p in body.input_files]
+
+    if body.instruction is not None:
+        merged["custom_llm_instructions"] = body.instruction
+    if body.output_dir is not None:
+        # Output folders may not exist yet (CLI will create). Still constrain to allowed roots.
+        merged["output_dir"] = _path_must_be_allowed_directory(
+            body.output_dir, must_exist=False
+        )
+    if body.input_dir is not None:
+        # Input dir should exist if provided.
+        merged["input_dir"] = _path_must_be_allowed_directory(
+            body.input_dir, must_exist=True
+        )
+    if body.ocr_method is not None:
+        merged["ocr_method"] = body.ocr_method
+    if body.pii_detector is not None:
+        merged["pii_detector"] = body.pii_detector
+
+    if body.overrides:
+        allowed = set(merged.keys())
+        for key, value in body.overrides.items():
+            if key not in allowed:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Unknown override key '{key}'. Must be a known CLI argument name.",
+                )
+            merged[key] = value
+
+    return merged
+
+
+def _run_cli_main(direct: dict[str, Any], gradio_api_name: str) -> AgentTaskResponse:
+    from cli_redact import main as cli_main
+
+    buf = io.StringIO()
+    old_stdout = sys.stdout
+    try:
+        sys.stdout = buf
+        cli_main(direct_mode_args=direct)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e)) from e
+    finally:
+        sys.stdout = old_stdout
+
+    log_excerpt = buf.getvalue()
+    if len(log_excerpt) > 8000:
+        log_excerpt = log_excerpt[-8000:]
+
+    return AgentTaskResponse(
+        status="completed",
+        gradio_api_name=gradio_api_name,
+        task=str(direct.get("task", "")),
+        output_dir=str(direct.get("output_dir", "")),
+        input_dir=str(direct.get("input_dir", "")),
+        message="cli_redact.main finished; see log_excerpt for console output",
+        log_excerpt=log_excerpt or None,
+    )
+
+
+@router.post(
+    "/redact_document",
+    response_model=AgentTaskResponse,
+    summary="redact_document (Gradio api_name)",
+    description=(
+        "Matches Gradio ``api_name='redact_document'``. "
+        "``python cli_redact.py --task redact --input_file ...``. "
+        "Optional ``instruction`` maps to ``custom_llm_instructions``. "
+        "OCR modes: 'Local OCR' | 'AWS Textract' | 'Local text'. "
+        "Specific local OCR engines are set via ``overrides.chosen_local_ocr_model`` "
+        f"(for example: {LOCAL_OCR_MODEL_OPTIONS}). "
+        "PII methods should use configured labels shown on the request schema."
+    ),
+)
+def post_redact_document(
+    body: AgentRedactDocumentRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    direct = _merge_redact_direct_mode(body)
+    return _run_cli_main(direct, "redact_document")
+
+
+@router.post(
+    "/redact_data",
+    response_model=AgentTaskResponse,
+    summary="redact_data (Gradio api_name)",
+    description=(
+        "Matches Gradio ``api_name='redact_data'``. Same CLI ``redact`` task as "
+        "/redact_document; use CSV/XLSX/DOCX paths for tabular/Word flows. "
+        "OCR modes: 'Local OCR' | 'AWS Textract' | 'Local text'. "
+        "Specific local OCR engines are set via ``overrides.chosen_local_ocr_model`` "
+        f"(for example: {LOCAL_OCR_MODEL_OPTIONS}). "
+        "PII methods should use configured labels shown on the request schema."
+    ),
+)
+def post_redact_data(
+    body: AgentRedactDataRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    direct = _merge_redact_direct_mode(body)
+    return _run_cli_main(direct, "redact_data")
+
+
+@router.post(
+    "/tasks/redact",
+    response_model=AgentTaskResponse,
+    summary="Legacy: same as /redact_document",
+    description="Deprecated alias; prefer POST /agent/redact_document.",
+    deprecated=True,
+    include_in_schema=True,
+)
+def post_tasks_redact_legacy(
+    body: AgentRedactDocumentRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    direct = _merge_redact_direct_mode(body)
+    return _run_cli_main(direct, "redact_document")
+
+
+class AgentFindDuplicatePagesRequest(BaseModel):
+    input_files: list[str] = Field(..., min_length=1)
+    similarity_threshold: Optional[float] = None
+    min_word_count: Optional[int] = None
+    min_consecutive_pages: Optional[int] = None
+    greedy_match: Optional[bool] = None
+    combine_pages: Optional[bool] = None
+    overrides: Optional[dict[str, Any]] = None
+
+
+@router.post(
+    "/find_duplicate_pages",
+    response_model=AgentTaskResponse,
+    summary="find_duplicate_pages (Gradio api_name)",
+    description="``cli_redact --task deduplicate --duplicate_type pages``.",
+)
+def post_find_duplicate_pages(
+    body: AgentFindDuplicatePagesRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from cli_redact import get_cli_default_args_dict
+
+    merged = get_cli_default_args_dict()
+    merged["task"] = "deduplicate"
+    merged["duplicate_type"] = "pages"
+    merged["input_file"] = [_path_must_be_allowed_file(p) for p in body.input_files]
+    if body.similarity_threshold is not None:
+        merged["similarity_threshold"] = body.similarity_threshold
+    if body.min_word_count is not None:
+        merged["min_word_count"] = body.min_word_count
+    if body.min_consecutive_pages is not None:
+        merged["min_consecutive_pages"] = body.min_consecutive_pages
+    if body.greedy_match is not None:
+        merged["greedy_match"] = "True" if body.greedy_match else "False"
+    if body.combine_pages is not None:
+        merged["combine_pages"] = "True" if body.combine_pages else "False"
+    if body.overrides:
+        allowed = set(merged.keys())
+        for k, v in body.overrides.items():
+            if k not in allowed:
+                raise HTTPException(400, f"Unknown override key: {k}")
+            merged[k] = v
+    return _run_cli_main(merged, "find_duplicate_pages")
+
+
+class AgentFindDuplicateTabularRequest(BaseModel):
+    input_files: list[str] = Field(..., min_length=1)
+    text_columns: Optional[list[str]] = None
+    similarity_threshold: Optional[float] = None
+    min_word_count: Optional[int] = None
+    overrides: Optional[dict[str, Any]] = None
+
+
+@router.post(
+    "/find_duplicate_tabular",
+    response_model=AgentTaskResponse,
+    summary="find_duplicate_tabular (Gradio api_name)",
+)
+def post_find_duplicate_tabular(
+    body: AgentFindDuplicateTabularRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from cli_redact import get_cli_default_args_dict
+
+    merged = get_cli_default_args_dict()
+    merged["task"] = "deduplicate"
+    merged["duplicate_type"] = "tabular"
+    merged["input_file"] = [_path_must_be_allowed_file(p) for p in body.input_files]
+    if body.text_columns is not None:
+        merged["text_columns"] = body.text_columns
+    if body.similarity_threshold is not None:
+        merged["similarity_threshold"] = body.similarity_threshold
+    if body.min_word_count is not None:
+        merged["min_word_count"] = body.min_word_count
+    if body.overrides:
+        allowed = set(merged.keys())
+        for k, v in body.overrides.items():
+            if k not in allowed:
+                raise HTTPException(400, f"Unknown override key: {k}")
+            merged[k] = v
+    return _run_cli_main(merged, "find_duplicate_tabular")
+
+
+class AgentSummariseDocumentRequest(BaseModel):
+    input_files: list[str] = Field(..., min_length=1)
+    summarisation_inference_method: Optional[str] = None
+    summarisation_format: Optional[str] = None
+    summarisation_context: Optional[str] = None
+    summarisation_additional_instructions: Optional[str] = None
+    overrides: Optional[dict[str, Any]] = None
+
+
+@router.post(
+    "/summarise_document",
+    response_model=AgentTaskResponse,
+    summary="summarise_document (Gradio api_name)",
+)
+def post_summarise_document(
+    body: AgentSummariseDocumentRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from cli_redact import get_cli_default_args_dict
+
+    merged = get_cli_default_args_dict()
+    merged["task"] = "summarise"
+    merged["input_file"] = [_path_must_be_allowed_file(p) for p in body.input_files]
+    if body.summarisation_inference_method is not None:
+        merged["summarisation_inference_method"] = body.summarisation_inference_method
+    if body.summarisation_format is not None:
+        merged["summarisation_format"] = body.summarisation_format
+    if body.summarisation_context is not None:
+        merged["summarisation_context"] = body.summarisation_context
+    if body.summarisation_additional_instructions is not None:
+        merged["summarisation_additional_instructions"] = (
+            body.summarisation_additional_instructions
+        )
+    if body.overrides:
+        allowed = set(merged.keys())
+        for k, v in body.overrides.items():
+            if k not in allowed:
+                raise HTTPException(400, f"Unknown override key: {k}")
+            merged[k] = v
+    return _run_cli_main(merged, "summarise_document")
+
+
+class AgentCombineReviewPdfsRequest(BaseModel):
+    input_files: list[str] = Field(..., min_length=2)
+    output_dir: Optional[str] = None
+
+
+@router.post(
+    "/combine_review_pdfs",
+    response_model=AgentTaskResponse,
+    summary="combine_review_pdfs (Gradio api_name)",
+)
+def post_combine_review_pdfs(
+    body: AgentCombineReviewPdfsRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from cli_redact import get_cli_default_args_dict
+
+    merged = get_cli_default_args_dict()
+    merged["task"] = "combine_review_pdfs"
+    merged["input_file"] = [_path_must_be_allowed_file(p) for p in body.input_files]
+    if body.output_dir is not None:
+        merged["output_dir"] = _path_must_be_allowed_directory(body.output_dir)
+    return _run_cli_main(merged, "combine_review_pdfs")
+
+
+class _NamedPath:
+    """merge_csv_files expects objects with a .name attribute (Gradio file-like)."""
+
+    __slots__ = ("name",)
+
+    def __init__(self, path: str) -> None:
+        self.name = path
+
+
+class AgentCombineReviewCsvsRequest(BaseModel):
+    input_files: list[str] = Field(..., min_length=1)
+    output_dir: Optional[str] = Field(
+        None, description="Defaults to config OUTPUT_FOLDER"
+    )
+
+
+class AgentApplyReviewRedactionsRequest(BaseModel):
+    """Headless parity with Gradio ``api_name='apply_review_redactions'`` (prepare + apply)."""
+
+    pdf_path: str = Field(
+        ...,
+        description="Path to the source PDF under allowed roots.",
+    )
+    review_csv_path: str = Field(
+        ...,
+        description=(
+            "Path to the review plan CSV; basename must contain '_review_file' "
+            "(e.g. mydoc_review_file.csv)."
+        ),
+    )
+    output_dir: Optional[str] = Field(
+        None,
+        description="Output directory (created if missing); defaults to OUTPUT_FOLDER.",
+    )
+    input_dir: Optional[str] = Field(
+        None,
+        description="Input/working directory for page images; defaults to INPUT_FOLDER.",
+    )
+    text_extract_method: Optional[str] = Field(
+        None,
+        description="OCR/text mode passed to prepare (defaults to CLI ocr_method).",
+    )
+    efficient_ocr: Optional[bool] = Field(
+        None,
+        description="If set, overrides EFFICIENT_OCR for the prepare step.",
+    )
+
+
+@router.post(
+    "/combine_review_csvs",
+    response_model=AgentTaskResponse,
+    summary="combine_review_csvs (Gradio api_name)",
+    description="Uses tools.helper_functions.merge_csv_files (not cli_redact).",
+)
+def post_combine_review_csvs(
+    body: AgentCombineReviewCsvsRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from tools.helper_functions import merge_csv_files
+
+    paths = [_NamedPath(_path_must_be_allowed_file(p)) for p in body.input_files]
+    out_dir = body.output_dir or OUTPUT_FOLDER
+    out_dir_resolved = _path_must_be_allowed_directory(str(out_dir), must_exist=True)
+    sep = "/" if not out_dir_resolved.endswith(("/", "\\")) else ""
+    out_files = merge_csv_files(paths, output_folder=out_dir_resolved + sep)
+    return AgentTaskResponse(
+        status="completed",
+        gradio_api_name="combine_review_csvs",
+        task="combine_review_csvs",
+        output_dir=out_dir_resolved,
+        input_dir="",
+        message="merge_csv_files completed",
+        output_paths=out_files,
+    )
+
+
+class AgentExportReviewRedactionOverlayRequest(BaseModel):
+    """Agent JSON body for the same overlay render as Gradio ``api_name='page_redaction_review_image'``."""
+
+    page_image_path: str = Field(
+        ...,
+        description="Path to page raster (PNG/JPEG) used as underlay; must be under allowed roots.",
+    )
+    boxes: List[Dict[str, Any]] = Field(
+        ...,
+        min_length=1,
+        description="Annotator-style boxes: label, color, xmin, ymin, xmax, ymax (normalized 0–1).",
+    )
+    page_number: int = Field(
+        1, ge=1, description="1-based page index for the output filename."
+    )
+    doc_base_name: str = Field(
+        "review",
+        description="Basename for output file (e.g. document name without extension).",
+    )
+    review_df_records: Optional[List[Dict[str, Any]]] = Field(
+        None,
+        description="Optional rows (include at least 'label') for stable label→line-pattern mapping.",
+    )
+    label_abbrev_chars: Optional[int] = Field(
+        None,
+        ge=0,
+        le=24,
+        description="Draw this many leading characters of each label on the image; omit to use REVIEW_OVERLAY_LABEL_ABBREV_CHARS from config (0 = off).",
+    )
+
+
+class AgentExportReviewPageOcrVisualisationRequest(BaseModel):
+    """Agent JSON body for the same OCR visualisation as Gradio ``api_name='page_ocr_review_image'``."""
+
+    page_image_path: str = Field(
+        ...,
+        description="Path to page raster (PNG/JPEG) used as underlay; must be under allowed roots.",
+    )
+    ocr_results: Dict[str, Any] = Field(
+        ...,
+        description="Word-level OCR results dict (line_key -> {words:[{text, bounding_box, conf, ...}]}).",
+    )
+    page_number: int = Field(
+        1, ge=1, description="1-based page index (used for naming)."
+    )
+    doc_base_name: str = Field(
+        "review",
+        description="Basename for output file (e.g. document name without extension).",
+    )
+
+
+@router.post(
+    "/export_review_redaction_overlay",
+    response_model=AgentTaskResponse,
+    summary="export_review_redaction_overlay (Agent API; Gradio api_name: page_redaction_review_image)",
+    description=(
+        "Renders hollow redaction outlines and a top-right legend on the page image; "
+        "writes ``redaction_overlay/{doc_base_name}_page{n}_redaction_overlay.jpg`` under OUTPUT_FOLDER "
+        "(scaled per REVIEW_OVERLAY_MAX_PIXELS, JPEG capped by REVIEW_OVERLAY_MAX_FILE_BYTES). "
+        "Uses ``tools.redaction_review.visualise_review_redaction_boxes``."
+    ),
+)
+def post_export_review_redaction_overlay(
+    body: AgentExportReviewRedactionOverlayRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    import pandas as pd
+
+    from tools.redaction_review import visualise_review_redaction_boxes
+
+    img_path = _path_must_be_allowed_file(body.page_image_path)
+    annotator: dict[str, Any] = {"image": img_path, "boxes": body.boxes}
+    review_df = (
+        pd.DataFrame(body.review_df_records)
+        if body.review_df_records
+        else pd.DataFrame()
+    )
+    out_folder_abs = os.path.realpath(
+        os.path.abspath(os.path.expanduser(str(OUTPUT_FOLDER)))
+    )
+    if not validate_path_safety(out_folder_abs):
+        raise HTTPException(status_code=400, detail="Unsafe OUTPUT_FOLDER path")
+    _must_be_under_allowed_roots(out_folder_abs, str(out_folder_abs))
+    try:
+        Path(out_folder_abs).mkdir(parents=True, exist_ok=True)
+    except OSError:
+        raise HTTPException(status_code=500, detail="Could not create OUTPUT_FOLDER")
+    out_folder = out_folder_abs
+
+    path = visualise_review_redaction_boxes(
+        annotator,
+        review_df=review_df,
+        output_folder=out_folder,
+        page_number=body.page_number,
+        doc_base_name=body.doc_base_name,
+        label_abbrev_chars=body.label_abbrev_chars,
+    )
+    if not path:
+        raise HTTPException(
+            status_code=500,
+            detail=(
+                "Could not produce overlay PNG (invalid image/boxes or write failed). "
+                "Ensure boxes are valid and the image loads."
+            ),
+        )
+    return AgentTaskResponse(
+        status="completed",
+        gradio_api_name="export_review_redaction_overlay",
+        task="export_review_redaction_overlay",
+        output_dir=out_folder,
+        input_dir="",
+        message="Redaction overlay PNG written",
+        output_paths=[path],
+    )
+
+
+@router.post(
+    "/export_review_page_ocr_visualisation",
+    response_model=AgentTaskResponse,
+    summary="export_review_page_ocr_visualisation (Agent API; Gradio api_name: page_ocr_review_image)",
+    description=(
+        "Renders a per-page OCR visualisation using tools.file_redaction.visualise_ocr_words_bounding_boxes; "
+        "writes under OUTPUT_FOLDER/review_ocr_visualisations/."
+    ),
+)
+def post_export_review_page_ocr_visualisation(
+    body: AgentExportReviewPageOcrVisualisationRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from PIL import Image
+
+    from tools.file_redaction import visualise_ocr_words_bounding_boxes
+
+    img_path = _path_must_be_allowed_file(body.page_image_path)
+
+    out_folder_abs = os.path.realpath(
+        os.path.abspath(os.path.expanduser(str(OUTPUT_FOLDER)))
+    )
+    if not validate_path_safety(out_folder_abs):
+        raise HTTPException(status_code=400, detail="Unsafe OUTPUT_FOLDER path")
+    _must_be_under_allowed_roots(out_folder_abs, str(out_folder_abs))
+    try:
+        Path(out_folder_abs).mkdir(parents=True, exist_ok=True)
+    except OSError:
+        raise HTTPException(status_code=500, detail="Could not create OUTPUT_FOLDER")
+    out_folder = out_folder_abs
+
+    safe_base = str(body.doc_base_name or "review")
+    image_name = f"{safe_base}_page{int(body.page_number)}.png"
+    log_paths: list[str] = []
+    try:
+        log_paths = visualise_ocr_words_bounding_boxes(
+            Image.open(img_path).convert("RGB"),
+            body.ocr_results,
+            image_name=image_name,
+            output_folder=out_folder,
+            visualisation_folder="review_ocr_visualisations",
+            add_legend=True,
+            log_files_output_paths=log_paths,
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e)) from e
+
+    if not log_paths:
+        raise HTTPException(
+            status_code=500,
+            detail="Could not produce OCR visualisation (invalid image/ocr_results or write failed).",
+        )
+    out_path = log_paths[-1]
+    return AgentTaskResponse(
+        status="completed",
+        gradio_api_name="export_review_page_ocr_visualisation",
+        task="export_review_page_ocr_visualisation",
+        output_dir=out_folder,
+        input_dir="",
+        message="OCR visualisation written",
+        output_paths=[out_path],
+    )
+
+
+def _gradio_only(api_name: str, detail: str) -> JSONResponse:
+    return JSONResponse(
+        status_code=501,
+        content={
+            "gradio_api_name": api_name,
+            "detail": detail,
+            "hint": (
+                "This flow is Gradio-session stateful. Call the named route on the "
+                "Gradio HTTP API, not /agent."
+            ),
+            "gradio_http": {
+                "discover_schema": "GET /gradio_api/info",
+                "start_call": f"POST /gradio_api/call/{api_name}",
+                "request_body_shape": '{"data": [<args in schema order>]}',
+                "poll": f"GET /gradio_api/call/{api_name}/{{event_id}}",
+            },
+            "gradio_client_notes": [
+                "Pass api_name explicitly; do not rely on inferring the endpoint from "
+                "Python function names (large Blocks apps will look ambiguous).",
+                "If predict() still cannot resolve the route, open GET /gradio_api/info "
+                "and use the numeric fn_index with gradio_client, or call the HTTP "
+                "endpoints directly.",
+                "The length of data must match the parameter list for this deployment; "
+                "copy order and types from /gradio_api/info.",
+            ],
+        },
+    )
+
+
+@router.post("/load_and_prepare_documents_or_data")
+def post_load_and_prepare_documents_or_data() -> JSONResponse:
+    return _gradio_only(
+        "load_and_prepare_documents_or_data",
+        "Preparation uses Gradio session state and prepare_image_or_pdf_with_efficient_ocr; no single CLI task.",
+    )
+
+
+@router.post(
+    "/apply_review_redactions",
+    response_model=AgentTaskResponse,
+    summary="apply_review_redactions (Gradio api_name)",
+    description=(
+        "Runs prepare_image_or_pdf_with_efficient_ocr([pdf, review_csv]) then "
+        "apply_redactions_to_review_df_and_files — same core pipeline as the Review tab, "
+        "without Gradio session state. Requires paths under allowed roots."
+    ),
+)
+def post_apply_review_redactions(
+    body: AgentApplyReviewRedactionsRequest,
+    _: None = Depends(_optional_agent_api_key),
+) -> AgentTaskResponse:
+    from tools.simplified_api import run_apply_review_redactions
+
+    pdf = _path_must_be_allowed_file(body.pdf_path)
+    csv = _path_must_be_allowed_file(body.review_csv_path)
+    out_dir: str | None = None
+    if body.output_dir is not None:
+        out_dir = _path_must_be_allowed_directory(body.output_dir, must_exist=False)
+    in_dir: str | None = None
+    if body.input_dir is not None:
+        in_dir = _path_must_be_allowed_directory(body.input_dir, must_exist=False)
+
+    try:
+        result = run_apply_review_redactions(
+            pdf_path=pdf,
+            review_csv_path=csv,
+            output_dir=out_dir,
+            input_dir=in_dir,
+            text_extract_method=body.text_extract_method,
+            efficient_ocr=body.efficient_ocr,
+        )
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e)) from e
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"apply_review_redactions failed: {e}",
+        ) from e
+
+    return AgentTaskResponse(
+        status="completed",
+        gradio_api_name="apply_review_redactions",
+        task="apply_review_redactions",
+        output_dir=result["output_dir"],
+        input_dir=result["input_dir"],
+        message=result["message"],
+        output_paths=result.get("output_paths"),
+    )
+
+
+@router.post("/word_level_ocr_text_search")
+def post_word_level_ocr_text_search() -> JSONResponse:
+    return _gradio_only(
+        "word_level_ocr_text_search",
+        "Search uses in-memory OCR dataframes in the UI session.",
+    )
+
+
+@router.get("/operations")
+def list_operations() -> dict[str, Any]:
+    return {
+        "gradio_api_names": list(GRADIO_API_NAMES),
+        "gradio_session_state_endpoints": {
+            "description": (
+                "These api_name values are exposed on the Gradio HTTP API but return "
+                "501 on /agent because they depend on in-memory Gradio state."
+            ),
+            "discover_schema": "GET /gradio_api/info",
+            "call_pattern": 'POST /gradio_api/call/<api_name> with JSON body {"data": [...]}',
+            "names": [
+                "load_and_prepare_documents_or_data",
+                "word_level_ocr_text_search",
+            ],
+        },
+        "routes": [
+            {
+                "gradio_api_name": "redact_document",
+                "method": "POST",
+                "path": "/agent/redact_document",
+                "implementation": "cli_redact task redact",
+                "notes": {
+                    "ocr_method": [
+                        "Local OCR",
+                        "AWS Textract",
+                        "Local text",
+                    ],
+                    "chosen_local_ocr_model_override": LOCAL_OCR_MODEL_OPTIONS,
+                    "pii_detector_recommended": [
+                        LOCAL_PII_OPTION,
+                        AWS_PII_OPTION,
+                        AWS_LLM_PII_OPTION,
+                        INFERENCE_SERVER_PII_OPTION,
+                        LOCAL_TRANSFORMERS_LLM_PII_OPTION,
+                        "None",
+                    ],
+                },
+            },
+            {
+                "gradio_api_name": "redact_data",
+                "method": "POST",
+                "path": "/agent/redact_data",
+                "implementation": "cli_redact task redact",
+                "notes": {
+                    "ocr_method": [
+                        "Local OCR",
+                        "AWS Textract",
+                        "Local text",
+                    ],
+                    "chosen_local_ocr_model_override": LOCAL_OCR_MODEL_OPTIONS,
+                    "pii_detector_recommended": [
+                        LOCAL_PII_OPTION,
+                        AWS_PII_OPTION,
+                        AWS_LLM_PII_OPTION,
+                        INFERENCE_SERVER_PII_OPTION,
+                        LOCAL_TRANSFORMERS_LLM_PII_OPTION,
+                        "None",
+                    ],
+                },
+            },
+            {
+                "gradio_api_name": "find_duplicate_pages",
+                "method": "POST",
+                "path": "/agent/find_duplicate_pages",
+                "implementation": "cli_redact deduplicate pages",
+            },
+            {
+                "gradio_api_name": "find_duplicate_tabular",
+                "method": "POST",
+                "path": "/agent/find_duplicate_tabular",
+                "implementation": "cli_redact deduplicate tabular",
+            },
+            {
+                "gradio_api_name": "summarise_document",
+                "method": "POST",
+                "path": "/agent/summarise_document",
+                "implementation": "cli_redact task summarise",
+            },
+            {
+                "gradio_api_name": "combine_review_pdfs",
+                "method": "POST",
+                "path": "/agent/combine_review_pdfs",
+                "implementation": "cli_redact combine_review_pdfs",
+            },
+            {
+                "gradio_api_name": "export_review_redaction_overlay",
+                "method": "POST",
+                "path": "/agent/export_review_redaction_overlay",
+                "implementation": "visualise_review_redaction_boxes",
+            },
+            {
+                "gradio_api_name": "export_review_page_ocr_visualisation",
+                "method": "POST",
+                "path": "/agent/export_review_page_ocr_visualisation",
+                "implementation": "visualise_ocr_words_bounding_boxes",
+            },
+            {
+                "gradio_api_name": "combine_review_csvs",
+                "method": "POST",
+                "path": "/agent/combine_review_csvs",
+                "implementation": "helper merge_csv_files",
+            },
+            {
+                "gradio_api_name": "load_and_prepare_documents_or_data",
+                "method": "POST",
+                "path": "/agent/load_and_prepare_documents_or_data",
+                "implementation": "not_implemented_http",
+            },
+            {
+                "gradio_api_name": "apply_review_redactions",
+                "method": "POST",
+                "path": "/agent/apply_review_redactions",
+                "implementation": "tools.simplified_api.run_apply_review_redactions",
+            },
+            {
+                "gradio_api_name": "word_level_ocr_text_search",
+                "method": "POST",
+                "path": "/agent/word_level_ocr_text_search",
+                "implementation": "not_implemented_http",
+            },
+        ],
+    }
+
+
+@router.get("/health")
+def agent_health() -> dict[str, str]:
+    return {"status": "ok", "service": "agent"}

cdk/app.py

@@ -0,0 +1,83 @@
+import os
+
+from aws_cdk import App, Environment
+from cdk_config import AWS_ACCOUNT_ID, AWS_REGION, RUN_USEAST_STACK, USE_CLOUDFRONT
+from cdk_functions import create_basic_config_env, load_context_from_file
+from cdk_stack import CdkStack, CdkStackCloudfront  # , CdkStackMain
+
+# Assuming these are still relevant for you
+from check_resources import CONTEXT_FILE, check_and_set_context
+
+# Initialize the CDK app
+app = App()
+
+# --- ENHANCED CONTEXT GENERATION AND LOADING ---
+# 1. Always ensure the old context file is removed before generation
+if os.path.exists(CONTEXT_FILE):
+    try:
+        os.remove(CONTEXT_FILE)
+        print(f"Removed stale context file: {CONTEXT_FILE}")
+    except OSError as e:
+        print(f"Warning: Could not remove old context file {CONTEXT_FILE}: {e}")
+        # Proceed anyway, check_and_set_context might handle overwriting
+
+# 2. Always run the pre-check script to generate fresh context
+print("Running pre-check script to generate application context...")
+try:
+    check_and_set_context()
+    if not os.path.exists(CONTEXT_FILE):
+        raise RuntimeError(
+            f"check_and_set_context() finished, but {CONTEXT_FILE} was not created."
+        )
+    print(f"Context generated successfully at {CONTEXT_FILE}.")
+except Exception as e:
+    raise RuntimeError(f"Failed to generate context via check_and_set_context(): {e}")
+
+if os.path.exists(CONTEXT_FILE):
+    load_context_from_file(app, CONTEXT_FILE)
+else:
+    raise RuntimeError(f"Could not find {CONTEXT_FILE}.")
+
+# Create basic config.env file that user can use to run the app later. Input is the folder it is saved into.
+create_basic_config_env("config")
+
+# Define the environment for the regional stack (where ALB resides)
+aws_env_regional = Environment(account=AWS_ACCOUNT_ID, region=AWS_REGION)
+
+# Create the regional stack (ALB, SGs, etc.)
+# regional_stack = CdkStack(app,
+#                           "RedactionStackSubnets",
+#                           env=aws_env_regional,
+#                           cross_region_references=True)
+
+# regional_stack_main = CdkStackMain(app,
+#                         "RedactionStackMain",
+#                         env=aws_env_regional,
+#                         private_subnets=regional_stack.params["private_subnets"],
+#                         private_route_tables=regional_stack.params["private_route_tables"],
+#                         public_subnets=regional_stack.params["public_subnets"],
+#                         public_route_tables=regional_stack.params["public_route_tables"],
+#                         cross_region_references=True)
+
+regional_stack = CdkStack(
+    app, "RedactionStack", env=aws_env_regional, cross_region_references=True
+)
+
+if USE_CLOUDFRONT == "True" and RUN_USEAST_STACK == "True":
+    # Define the environment for the CloudFront stack (always us-east-1 for CF-level resources like WAFv2 WebACLs for CF)
+    aws_env_us_east_1 = Environment(account=AWS_ACCOUNT_ID, region="us-east-1")
+
+    # Create the CloudFront stack, passing the outputs from the regional stack
+    cloudfront_stack = CdkStackCloudfront(
+        app,
+        "RedactionStackCloudfront",
+        env=aws_env_us_east_1,
+        alb_arn=regional_stack.params["alb_arn_output"],
+        alb_sec_group_id=regional_stack.params["alb_security_group_id"],
+        alb_dns_name=regional_stack.params["alb_dns_name"],
+        cross_region_references=True,
+    )
+
+
+# Synthesize the CloudFormation template
+app.synth(validate_on_synthesis=True)

cdk/cdk_config.py

@@ -0,0 +1,362 @@
+import os
+import tempfile
+
+from dotenv import load_dotenv
+
+# Set or retrieve configuration variables for CDK redaction deployment
+
+
+def convert_string_to_boolean(value: str) -> bool:
+    """Convert string to boolean, handling various formats."""
+    if isinstance(value, bool):
+        return value
+    elif value in ["True", "1", "true", "TRUE"]:
+        return True
+    elif value in ["False", "0", "false", "FALSE"]:
+        return False
+    else:
+        raise ValueError(f"Invalid boolean value: {value}")
+
+
+def get_or_create_env_var(var_name: str, default_value: str, print_val: bool = False):
+    """
+    Get an environmental variable, and set it to a default value if it doesn't exist
+    """
+    # Get the environment variable if it exists
+    value = os.environ.get(var_name)
+
+    # If it doesn't exist, set the environment variable to the default value
+    if value is None:
+        os.environ[var_name] = default_value
+        value = default_value
+
+    if print_val is True:
+        print(f"The value of {var_name} is {value}")
+
+    return value
+
+
+def ensure_folder_exists(output_folder: str):
+    """Checks if the specified folder exists, creates it if not."""
+
+    if not os.path.exists(output_folder):
+        # Create the folder if it doesn't exist
+        os.makedirs(output_folder, exist_ok=True)
+        print(f"Created the {output_folder} folder.")
+    else:
+        print(f"The {output_folder} folder already exists.")
+
+
+def add_folder_to_path(folder_path: str):
+    """
+    Check if a folder exists on your system. If so, get the absolute path and then add it to the system Path variable if it doesn't already exist. Function is only relevant for locally-created executable files based on this app (when using pyinstaller it creates a _internal folder that contains tesseract and poppler. These need to be added to the system path to enable the app to run)
+    """
+
+    if os.path.exists(folder_path) and os.path.isdir(folder_path):
+        print(folder_path, "folder exists.")
+
+        # Resolve relative path to absolute path
+        absolute_path = os.path.abspath(folder_path)
+
+        current_path = os.environ["PATH"]
+        if absolute_path not in current_path.split(os.pathsep):
+            full_path_extension = absolute_path + os.pathsep + current_path
+            os.environ["PATH"] = full_path_extension
+            # print(f"Updated PATH with: ", full_path_extension)
+        else:
+            print(f"Directory {folder_path} already exists in PATH.")
+    else:
+        print(f"Folder not found at {folder_path} - not added to PATH")
+
+
+###
+# LOAD CONFIG FROM ENV FILE
+###
+CONFIG_FOLDER = get_or_create_env_var("CONFIG_FOLDER", "config/")
+
+ensure_folder_exists(CONFIG_FOLDER)
+
+# If you have an aws_config env file in the config folder, you can load in app variables this way, e.g. 'config/cdk_config.env'
+CDK_CONFIG_PATH = get_or_create_env_var(
+    "CDK_CONFIG_PATH", "config/cdk_config.env"
+)  # e.g. config/cdk_config.env
+
+if CDK_CONFIG_PATH:
+    if os.path.exists(CDK_CONFIG_PATH):
+        print(f"Loading CDK variables from config file {CDK_CONFIG_PATH}")
+        load_dotenv(CDK_CONFIG_PATH)
+    else:
+        print("CDK config file not found at location:", CDK_CONFIG_PATH)
+
+###
+# AWS OPTIONS
+###
+AWS_REGION = get_or_create_env_var("AWS_REGION", "")
+AWS_ACCOUNT_ID = get_or_create_env_var("AWS_ACCOUNT_ID", "")
+
+###
+# CDK OPTIONS
+###
+CDK_PREFIX = get_or_create_env_var("CDK_PREFIX", "")
+CONTEXT_FILE = get_or_create_env_var(
+    "CONTEXT_FILE", "cdk.context.json"
+)  # Define the CDK output context file name
+CDK_FOLDER = get_or_create_env_var(
+    "CDK_FOLDER", ""
+)  # FULL_PATH_TO_CDK_FOLDER_HERE (with forward slash)
+RUN_USEAST_STACK = get_or_create_env_var("RUN_USEAST_STACK", "False")
+
+### VPC and connections
+VPC_NAME = get_or_create_env_var("VPC_NAME", "")
+NEW_VPC_DEFAULT_NAME = get_or_create_env_var("NEW_VPC_DEFAULT_NAME", f"{CDK_PREFIX}vpc")
+NEW_VPC_CIDR = get_or_create_env_var("NEW_VPC_CIDR", "")  # "10.0.0.0/24"
+
+
+EXISTING_IGW_ID = get_or_create_env_var("EXISTING_IGW_ID", "")
+SINGLE_NAT_GATEWAY_ID = get_or_create_env_var("SINGLE_NAT_GATEWAY_ID", "")
+
+### SUBNETS / ROUTE TABLES / NAT GATEWAY
+PUBLIC_SUBNETS_TO_USE = get_or_create_env_var(
+    "PUBLIC_SUBNETS_TO_USE", ""
+)  # e.g. ['PublicSubnet1', 'PublicSubnet2']
+PUBLIC_SUBNET_CIDR_BLOCKS = get_or_create_env_var(
+    "PUBLIC_SUBNET_CIDR_BLOCKS", ""
+)  # e.g. ["10.0.1.0/24", "10.0.2.0/24"]
+PUBLIC_SUBNET_AVAILABILITY_ZONES = get_or_create_env_var(
+    "PUBLIC_SUBNET_AVAILABILITY_ZONES", ""
+)  # e.g. ["eu-east-1b", "eu-east1b"]
+
+PRIVATE_SUBNETS_TO_USE = get_or_create_env_var(
+    "PRIVATE_SUBNETS_TO_USE", ""
+)  # e.g. ['PrivateSubnet1', 'PrivateSubnet2']
+PRIVATE_SUBNET_CIDR_BLOCKS = get_or_create_env_var(
+    "PRIVATE_SUBNET_CIDR_BLOCKS", ""
+)  # e.g. ["10.0.1.0/24", "10.0.2.0/24"]
+PRIVATE_SUBNET_AVAILABILITY_ZONES = get_or_create_env_var(
+    "PRIVATE_SUBNET_AVAILABILITY_ZONES", ""
+)  # e.g. ["eu-east-1b", "eu-east1b"]
+
+ROUTE_TABLE_BASE_NAME = get_or_create_env_var(
+    "ROUTE_TABLE_BASE_NAME", f"{CDK_PREFIX}PrivateRouteTable"
+)
+NAT_GATEWAY_EIP_NAME = get_or_create_env_var(
+    "NAT_GATEWAY_EIP_NAME", f"{CDK_PREFIX}NatGatewayEip"
+)
+NAT_GATEWAY_NAME = get_or_create_env_var("NAT_GATEWAY_NAME", f"{CDK_PREFIX}NatGateway")
+
+# IAM roles
+AWS_MANAGED_TASK_ROLES_LIST = get_or_create_env_var(
+    "AWS_MANAGED_TASK_ROLES_LIST",
+    '["AmazonCognitoReadOnly", "service-role/AmazonECSTaskExecutionRolePolicy", "AmazonS3FullAccess", "AmazonTextractFullAccess", "ComprehendReadOnly", "AmazonDynamoDBFullAccess", "service-role/AWSAppSyncPushToCloudWatchLogs", "AmazonBedrockFullAccess"]',
+)
+POLICY_FILE_LOCATIONS = get_or_create_env_var(
+    "POLICY_FILE_LOCATIONS", ""
+)  # e.g. '["config/sts_permissions.json"]'
+POLICY_FILE_ARNS = get_or_create_env_var("POLICY_FILE_ARNS", "")
+
+# GITHUB REPO
+GITHUB_REPO_USERNAME = get_or_create_env_var("GITHUB_REPO_USERNAME", "seanpedrick-case")
+GITHUB_REPO_NAME = get_or_create_env_var("GITHUB_REPO_NAME", "doc_redaction")
+GITHUB_REPO_BRANCH = get_or_create_env_var("GITHUB_REPO_BRANCH", "main")
+
+### CODEBUILD
+CODEBUILD_ROLE_NAME = get_or_create_env_var(
+    "CODEBUILD_ROLE_NAME", f"{CDK_PREFIX}CodeBuildRole"
+)
+CODEBUILD_PROJECT_NAME = get_or_create_env_var(
+    "CODEBUILD_PROJECT_NAME", f"{CDK_PREFIX}CodeBuildProject"
+)
+
+### ECR
+ECR_REPO_NAME = get_or_create_env_var(
+    "ECR_REPO_NAME", "doc-redaction"
+)  # Beware - cannot have underscores and must be lower case
+ECR_CDK_REPO_NAME = get_or_create_env_var(
+    "ECR_CDK_REPO_NAME", f"{CDK_PREFIX}{ECR_REPO_NAME}".lower()
+)
+
+### S3
+S3_LOG_CONFIG_BUCKET_NAME = get_or_create_env_var(
+    "S3_LOG_CONFIG_BUCKET_NAME", f"{CDK_PREFIX}s3-logs".lower()
+)  # S3 bucket names need to be lower case
+S3_OUTPUT_BUCKET_NAME = get_or_create_env_var(
+    "S3_OUTPUT_BUCKET_NAME", f"{CDK_PREFIX}s3-output".lower()
+)
+
+### KMS KEYS FOR S3 AND SECRETS MANAGER
+USE_CUSTOM_KMS_KEY = get_or_create_env_var("USE_CUSTOM_KMS_KEY", "1")
+CUSTOM_KMS_KEY_NAME = get_or_create_env_var(
+    "CUSTOM_KMS_KEY_NAME", f"alias/{CDK_PREFIX}kms-key".lower()
+)
+
+### ECS
+FARGATE_TASK_DEFINITION_NAME = get_or_create_env_var(
+    "FARGATE_TASK_DEFINITION_NAME", f"{CDK_PREFIX}FargateTaskDefinition"
+)
+TASK_DEFINITION_FILE_LOCATION = get_or_create_env_var(
+    "TASK_DEFINITION_FILE_LOCATION", CDK_FOLDER + CONFIG_FOLDER + "task_definition.json"
+)
+
+CLUSTER_NAME = get_or_create_env_var("CLUSTER_NAME", f"{CDK_PREFIX}Cluster")
+ECS_SERVICE_NAME = get_or_create_env_var("ECS_SERVICE_NAME", f"{CDK_PREFIX}ECSService")
+ECS_TASK_ROLE_NAME = get_or_create_env_var(
+    "ECS_TASK_ROLE_NAME", f"{CDK_PREFIX}TaskRole"
+)
+ECS_TASK_EXECUTION_ROLE_NAME = get_or_create_env_var(
+    "ECS_TASK_EXECUTION_ROLE_NAME", f"{CDK_PREFIX}ExecutionRole"
+)
+ECS_SECURITY_GROUP_NAME = get_or_create_env_var(
+    "ECS_SECURITY_GROUP_NAME", f"{CDK_PREFIX}SecurityGroupECS"
+)
+ECS_LOG_GROUP_NAME = get_or_create_env_var(
+    "ECS_LOG_GROUP_NAME", f"/ecs/{ECS_SERVICE_NAME}-logs".lower()
+)
+
+ECS_TASK_CPU_SIZE = get_or_create_env_var("ECS_TASK_CPU_SIZE", "1024")
+ECS_TASK_MEMORY_SIZE = get_or_create_env_var("ECS_TASK_MEMORY_SIZE", "4096")
+ECS_USE_FARGATE_SPOT = get_or_create_env_var("USE_FARGATE_SPOT", "False")
+ECS_READ_ONLY_FILE_SYSTEM = get_or_create_env_var("ECS_READ_ONLY_FILE_SYSTEM", "True")
+
+### Cognito
+COGNITO_USER_POOL_NAME = get_or_create_env_var(
+    "COGNITO_USER_POOL_NAME", f"{CDK_PREFIX}UserPool"
+)
+COGNITO_USER_POOL_CLIENT_NAME = get_or_create_env_var(
+    "COGNITO_USER_POOL_CLIENT_NAME", f"{CDK_PREFIX}UserPoolClient"
+)
+COGNITO_USER_POOL_CLIENT_SECRET_NAME = get_or_create_env_var(
+    "COGNITO_USER_POOL_CLIENT_SECRET_NAME", f"{CDK_PREFIX}ParamCognitoSecret"
+)
+COGNITO_USER_POOL_DOMAIN_PREFIX = get_or_create_env_var(
+    "COGNITO_USER_POOL_DOMAIN_PREFIX", "redaction-app-domain"
+)  # Should change this to something unique or you'll probably hit an error
+
+COGNITO_REFRESH_TOKEN_VALIDITY = int(
+    get_or_create_env_var("COGNITO_REFRESH_TOKEN_VALIDITY", "480")
+)  # Minutes
+COGNITO_ID_TOKEN_VALIDITY = int(
+    get_or_create_env_var("COGNITO_ID_TOKEN_VALIDITY", "60")
+)  # Minutes
+COGNITO_ACCESS_TOKEN_VALIDITY = int(
+    get_or_create_env_var("COGNITO_ACCESS_TOKEN_VALIDITY", "60")
+)  # Minutes
+
+# Application load balancer
+ALB_NAME = get_or_create_env_var(
+    "ALB_NAME", f"{CDK_PREFIX}Alb"[-32:]
+)  # Application load balancer name can be max 32 characters, so taking the last 32 characters of the suggested name
+ALB_NAME_SECURITY_GROUP_NAME = get_or_create_env_var(
+    "ALB_SECURITY_GROUP_NAME", f"{CDK_PREFIX}SecurityGroupALB"
+)
+ALB_TARGET_GROUP_NAME = get_or_create_env_var(
+    "ALB_TARGET_GROUP_NAME", f"{CDK_PREFIX}-tg"[-32:]
+)  # Max 32 characters
+EXISTING_LOAD_BALANCER_ARN = get_or_create_env_var("EXISTING_LOAD_BALANCER_ARN", "")
+EXISTING_LOAD_BALANCER_DNS = get_or_create_env_var(
+    "EXISTING_LOAD_BALANCER_ARN", "placeholder_load_balancer_dns.net"
+)
+
+## CLOUDFRONT
+USE_CLOUDFRONT = get_or_create_env_var("USE_CLOUDFRONT", "True")
+CLOUDFRONT_PREFIX_LIST_ID = get_or_create_env_var(
+    "CLOUDFRONT_PREFIX_LIST_ID", "pl-93a247fa"
+)
+CLOUDFRONT_GEO_RESTRICTION = get_or_create_env_var(
+    "CLOUDFRONT_GEO_RESTRICTION", ""
+)  # A country that Cloudfront restricts access to. See here: https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/georestrictions.html
+CLOUDFRONT_DISTRIBUTION_NAME = get_or_create_env_var(
+    "CLOUDFRONT_DISTRIBUTION_NAME", f"{CDK_PREFIX}CfDist"
+)
+CLOUDFRONT_DOMAIN = get_or_create_env_var(
+    "CLOUDFRONT_DOMAIN", "cloudfront_placeholder.net"
+)
+
+
+# Certificate for Application load balancer (optional, for HTTPS and logins through the ALB)
+ACM_SSL_CERTIFICATE_ARN = get_or_create_env_var("ACM_SSL_CERTIFICATE_ARN", "")
+SSL_CERTIFICATE_DOMAIN = get_or_create_env_var(
+    "SSL_CERTIFICATE_DOMAIN", ""
+)  # e.g. example.com or www.example.com
+
+# This should be the CloudFront domain, the domain linked to your ACM certificate, or the DNS of your application load balancer in console afterwards
+if USE_CLOUDFRONT == "True":
+    COGNITO_REDIRECTION_URL = get_or_create_env_var(
+        "COGNITO_REDIRECTION_URL", "https://" + CLOUDFRONT_DOMAIN
+    )
+elif SSL_CERTIFICATE_DOMAIN:
+    COGNITO_REDIRECTION_URL = get_or_create_env_var(
+        "COGNITO_REDIRECTION_URL", "https://" + SSL_CERTIFICATE_DOMAIN
+    )
+else:
+    COGNITO_REDIRECTION_URL = get_or_create_env_var(
+        "COGNITO_REDIRECTION_URL", "https://" + EXISTING_LOAD_BALANCER_DNS
+    )
+
+# Custom headers e.g. if routing traffic through Cloudfront
+CUSTOM_HEADER = get_or_create_env_var(
+    "CUSTOM_HEADER", ""
+)  # Retrieving or setting CUSTOM_HEADER
+CUSTOM_HEADER_VALUE = get_or_create_env_var(
+    "CUSTOM_HEADER_VALUE", ""
+)  # Retrieving or setting CUSTOM_HEADER_VALUE
+
+# Firewall on top of load balancer
+LOAD_BALANCER_WEB_ACL_NAME = get_or_create_env_var(
+    "LOAD_BALANCER_WEB_ACL_NAME", f"{CDK_PREFIX}alb-web-acl"
+)
+
+# Firewall on top of CloudFront
+WEB_ACL_NAME = get_or_create_env_var("WEB_ACL_NAME", f"{CDK_PREFIX}cloudfront-web-acl")
+
+###
+# File I/O options
+###
+
+OUTPUT_FOLDER = get_or_create_env_var("GRADIO_OUTPUT_FOLDER", "output/")  # 'output/'
+INPUT_FOLDER = get_or_create_env_var("GRADIO_INPUT_FOLDER", "input/")  # 'input/'
+
+# Allow for files to be saved in a temporary folder for increased security in some instances
+if OUTPUT_FOLDER == "TEMP" or INPUT_FOLDER == "TEMP":
+    # Create a temporary directory
+    with tempfile.TemporaryDirectory() as temp_dir:
+        print(f"Temporary directory created at: {temp_dir}")
+
+        if OUTPUT_FOLDER == "TEMP":
+            OUTPUT_FOLDER = temp_dir + "/"
+        if INPUT_FOLDER == "TEMP":
+            INPUT_FOLDER = temp_dir + "/"
+
+###
+# LOGGING OPTIONS
+###
+
+SAVE_LOGS_TO_CSV = get_or_create_env_var("SAVE_LOGS_TO_CSV", "True")
+
+### DYNAMODB logs. Whether to save to DynamoDB, and the headers of the table
+SAVE_LOGS_TO_DYNAMODB = get_or_create_env_var("SAVE_LOGS_TO_DYNAMODB", "True")
+ACCESS_LOG_DYNAMODB_TABLE_NAME = get_or_create_env_var(
+    "ACCESS_LOG_DYNAMODB_TABLE_NAME", f"{CDK_PREFIX}dynamodb-access-logs".lower()
+)
+FEEDBACK_LOG_DYNAMODB_TABLE_NAME = get_or_create_env_var(
+    "FEEDBACK_LOG_DYNAMODB_TABLE_NAME", f"{CDK_PREFIX}dynamodb-feedback-logs".lower()
+)
+USAGE_LOG_DYNAMODB_TABLE_NAME = get_or_create_env_var(
+    "USAGE_LOG_DYNAMODB_TABLE_NAME", f"{CDK_PREFIX}dynamodb-usage-logs".lower()
+)
+
+###
+# REDACTION OPTIONS
+###
+
+# Get some environment variables and Launch the Gradio app
+COGNITO_AUTH = get_or_create_env_var("COGNITO_AUTH", "0")
+
+GRADIO_SERVER_PORT = int(get_or_create_env_var("GRADIO_SERVER_PORT", "7860"))
+
+###
+# WHOLE DOCUMENT API OPTIONS
+###
+
+DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS = get_or_create_env_var(
+    "DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS", "7"
+)  # How many days into the past should whole document Textract jobs be displayed? After that, the data is not deleted from the Textract jobs csv, but it is just filtered out. Included to align with S3 buckets where the file outputs will be automatically deleted after X days.

cdk/cdk_functions.py

@@ -0,0 +1,1482 @@
+import ipaddress
+import json
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+import boto3
+import pandas as pd
+from aws_cdk import App, CfnOutput, CfnTag, Tags
+from aws_cdk import aws_cognito as cognito
+from aws_cdk import aws_ec2 as ec2
+from aws_cdk import aws_elasticloadbalancingv2 as elb
+from aws_cdk import aws_elasticloadbalancingv2_actions as elb_act
+from aws_cdk import aws_iam as iam
+from aws_cdk import aws_wafv2 as wafv2
+from botocore.exceptions import ClientError
+from cdk_config import (
+    ACCESS_LOG_DYNAMODB_TABLE_NAME,
+    AWS_REGION,
+    FEEDBACK_LOG_DYNAMODB_TABLE_NAME,
+    NAT_GATEWAY_EIP_NAME,
+    POLICY_FILE_LOCATIONS,
+    PRIVATE_SUBNET_AVAILABILITY_ZONES,
+    PRIVATE_SUBNET_CIDR_BLOCKS,
+    PRIVATE_SUBNETS_TO_USE,
+    PUBLIC_SUBNET_AVAILABILITY_ZONES,
+    PUBLIC_SUBNET_CIDR_BLOCKS,
+    PUBLIC_SUBNETS_TO_USE,
+    S3_LOG_CONFIG_BUCKET_NAME,
+    S3_OUTPUT_BUCKET_NAME,
+    USAGE_LOG_DYNAMODB_TABLE_NAME,
+)
+from constructs import Construct
+from dotenv import set_key
+
+
+# --- Function to load context from file ---
+def load_context_from_file(app: App, file_path: str):
+    if os.path.exists(file_path):
+        with open(file_path, "r") as f:
+            context_data = json.load(f)
+            for key, value in context_data.items():
+                app.node.set_context(key, value)
+            print(f"Loaded context from {file_path}")
+    else:
+        print(f"Context file not found: {file_path}")
+
+
+# --- Helper to parse environment variables into lists ---
+def _get_env_list(env_var_name: str) -> List[str]:
+    """Parses a comma-separated environment variable into a list of strings."""
+    value = env_var_name[1:-1].strip().replace('"', "").replace("'", "")
+    if not value:
+        return []
+    # Split by comma and filter out any empty strings that might result from extra commas
+    return [s.strip() for s in value.split(",") if s.strip()]
+
+
+# 1. Try to load CIDR/AZs from environment variables
+if PUBLIC_SUBNETS_TO_USE:
+    PUBLIC_SUBNETS_TO_USE = _get_env_list(PUBLIC_SUBNETS_TO_USE)
+if PRIVATE_SUBNETS_TO_USE:
+    PRIVATE_SUBNETS_TO_USE = _get_env_list(PRIVATE_SUBNETS_TO_USE)
+
+if PUBLIC_SUBNET_CIDR_BLOCKS:
+    PUBLIC_SUBNET_CIDR_BLOCKS = _get_env_list("PUBLIC_SUBNET_CIDR_BLOCKS")
+if PUBLIC_SUBNET_AVAILABILITY_ZONES:
+    PUBLIC_SUBNET_AVAILABILITY_ZONES = _get_env_list("PUBLIC_SUBNET_AVAILABILITY_ZONES")
+if PRIVATE_SUBNET_CIDR_BLOCKS:
+    PRIVATE_SUBNET_CIDR_BLOCKS = _get_env_list("PRIVATE_SUBNET_CIDR_BLOCKS")
+if PRIVATE_SUBNET_AVAILABILITY_ZONES:
+    PRIVATE_SUBNET_AVAILABILITY_ZONES = _get_env_list(
+        "PRIVATE_SUBNET_AVAILABILITY_ZONES"
+    )
+
+if POLICY_FILE_LOCATIONS:
+    POLICY_FILE_LOCATIONS = _get_env_list(POLICY_FILE_LOCATIONS)
+
+
+def check_for_existing_role(role_name: str):
+    try:
+        iam = boto3.client("iam")
+        # iam.get_role(RoleName=role_name)
+
+        response = iam.get_role(RoleName=role_name)
+        role = response["Role"]["Arn"]
+
+        print("Response Role:", role)
+
+        return True, role, ""
+    except iam.exceptions.NoSuchEntityException:
+        return False, "", ""
+    except Exception as e:
+        raise Exception("Getting information on IAM role failed due to:", e)
+
+
+from typing import List
+
+# Assume POLICY_FILE_LOCATIONS is defined globally or passed as a default
+# For example:
+# POLICY_FILE_LOCATIONS = ["./policies/my_read_policy.json", "./policies/my_write_policy.json"]
+
+
+def add_statement_to_policy(role: iam.IRole, policy_document: Dict[str, Any]):
+    """
+    Adds individual policy statements from a parsed policy document to a CDK Role.
+
+    Args:
+        role: The CDK Role construct to attach policies to.
+        policy_document: A Python dictionary representing an IAM policy document.
+    """
+    # Ensure the loaded JSON is a valid policy document structure
+    if "Statement" not in policy_document or not isinstance(
+        policy_document["Statement"], list
+    ):
+        print("Warning: Policy document does not contain a 'Statement' list. Skipping.")
+        return  # Do not return role, just log and exit
+
+    for statement_dict in policy_document["Statement"]:
+        try:
+            # Create a CDK PolicyStatement from the dictionary
+            cdk_policy_statement = iam.PolicyStatement.from_json(statement_dict)
+
+            # Add the policy statement to the role
+            role.add_to_policy(cdk_policy_statement)
+            print(f"  - Added statement: {statement_dict.get('Sid', 'No Sid')}")
+        except Exception as e:
+            print(
+                f"Warning: Could not process policy statement: {statement_dict}. Error: {e}"
+            )
+
+
+def add_custom_policies(
+    scope: Construct,  # Not strictly used here, but good practice if you expand to ManagedPolicies
+    role: iam.IRole,
+    policy_file_locations: Optional[List[str]] = None,
+    custom_policy_text: Optional[str] = None,
+) -> iam.IRole:
+    """
+    Loads custom policies from JSON files or a string and attaches them to a CDK Role.
+
+    Args:
+        scope: The scope in which to define constructs (if needed, e.g., for iam.ManagedPolicy).
+        role: The CDK Role construct to attach policies to.
+        policy_file_locations: List of file paths to JSON policy documents.
+        custom_policy_text: A JSON string representing a policy document.
+
+    Returns:
+        The modified CDK Role construct.
+    """
+    if policy_file_locations is None:
+        policy_file_locations = []
+
+    current_source = "unknown source"  # For error messages
+
+    try:
+        if policy_file_locations:
+            print(f"Attempting to add policies from files to role {role.node.id}...")
+            for path in policy_file_locations:
+                current_source = f"file: {path}"
+                try:
+                    with open(path, "r") as f:
+                        policy_document = json.load(f)
+                    print(f"Processing policy from {current_source}...")
+                    add_statement_to_policy(role, policy_document)
+                except FileNotFoundError:
+                    print(f"Warning: Policy file not found at {path}. Skipping.")
+                except json.JSONDecodeError as e:
+                    print(
+                        f"Warning: Invalid JSON in policy file {path}: {e}. Skipping."
+                    )
+                except Exception as e:
+                    print(
+                        f"An unexpected error occurred processing policy from {path}: {e}. Skipping."
+                    )
+
+        if custom_policy_text:
+            current_source = "custom policy text string"
+            print(
+                f"Attempting to add policy from custom text to role {role.node.id}..."
+            )
+            try:
+                # *** FIX: Parse the JSON string into a Python dictionary ***
+                policy_document = json.loads(custom_policy_text)
+                print(f"Processing policy from {current_source}...")
+                add_statement_to_policy(role, policy_document)
+            except json.JSONDecodeError as e:
+                print(f"Warning: Invalid JSON in custom_policy_text: {e}. Skipping.")
+            except Exception as e:
+                print(
+                    f"An unexpected error occurred processing policy from custom_policy_text: {e}. Skipping."
+                )
+
+        # You might want a final success message, but individual processing messages are also good.
+        print(f"Finished processing custom policies for role {role.node.id}.")
+
+    except Exception as e:
+        print(
+            f"An unhandled error occurred during policy addition for {current_source}: {e}"
+        )
+
+    return role
+
+
+# Import the S3 Bucket class if you intend to return a CDK object later
+# from aws_cdk import aws_s3 as s3
+
+
+def check_s3_bucket_exists(
+    bucket_name: str,
+):  # Return type hint depends on what you return
+    """
+    Checks if an S3 bucket with the given name exists and is accessible.
+
+    Args:
+        bucket_name: The name of the S3 bucket to check.
+
+    Returns:
+        A tuple: (bool indicating existence, optional S3 Bucket object or None)
+        Note: Returning a Boto3 S3 Bucket object from here is NOT ideal
+              for direct use in CDK. You'll likely only need the boolean result
+              or the bucket name for CDK lookups/creations.
+              For this example, let's return the boolean and the name.
+    """
+    s3_client = boto3.client("s3")
+    try:
+        # Use head_bucket to check for existence and access
+        s3_client.head_bucket(Bucket=bucket_name)
+        print(f"Bucket '{bucket_name}' exists and is accessible.")
+        return True, bucket_name  # Return True and the bucket name
+
+    except ClientError as e:
+        # If a ClientError occurs, check the error code.
+        # '404' means the bucket does not exist.
+        # '403' means the bucket exists but you don't have permission.
+        error_code = e.response["Error"]["Code"]
+        if error_code == "404":
+            print(f"Bucket '{bucket_name}' does not exist.")
+            return False, None
+        elif error_code == "403":
+            # The bucket exists, but you can't access it.
+            # Depending on your requirements, this might be treated as "exists"
+            # or "not accessible for our purpose". For checking existence,
+            # we'll say it exists here, but note the permission issue.
+            # NOTE - when I tested this, it was returning 403 even for buckets that don't exist. So I will return False instead
+            print(
+                f"Bucket '{bucket_name}' returned 403, which indicates it may exist but is not accessible due to permissions, or that it doesn't exist. Returning False for existence just in case."
+            )
+            return False, bucket_name  # It exists, even if not accessible
+        else:
+            # For other errors, it's better to raise the exception
+            # to indicate something unexpected happened.
+            print(
+                f"An unexpected AWS ClientError occurred checking bucket '{bucket_name}': {e}"
+            )
+            # Decide how to handle other errors - raising might be safer
+            raise  # Re-raise the original exception
+    except Exception as e:
+        print(
+            f"An unexpected non-ClientError occurred checking bucket '{bucket_name}': {e}"
+        )
+        # Decide how to handle other errors
+        raise  # Re-raise the original exception
+
+
+# Example usage in your check_resources.py:
+# exists, bucket_name_if_exists = check_s3_bucket_exists(log_bucket_name)
+# context_data[f"exists:{log_bucket_name}"] = exists
+# # You don't necessarily need to store the name in context if using from_bucket_name
+
+
+# Delete an S3 bucket
+def delete_s3_bucket(bucket_name: str):
+    s3 = boto3.client("s3")
+
+    try:
+        # List and delete all objects
+        response = s3.list_object_versions(Bucket=bucket_name)
+        versions = response.get("Versions", []) + response.get("DeleteMarkers", [])
+        for version in versions:
+            s3.delete_object(
+                Bucket=bucket_name, Key=version["Key"], VersionId=version["VersionId"]
+            )
+
+        # Delete the bucket
+        s3.delete_bucket(Bucket=bucket_name)
+        return {"Status": "SUCCESS"}
+    except Exception as e:
+        return {"Status": "FAILED", "Reason": str(e)}
+
+
+# Function to get subnet ID from subnet name
+def get_subnet_id(vpc: str, ec2_client: str, subnet_name: str):
+    response = ec2_client.describe_subnets(
+        Filters=[{"Name": "vpc-id", "Values": [vpc.vpc_id]}]
+    )
+
+    for subnet in response["Subnets"]:
+        if subnet["Tags"] and any(
+            tag["Key"] == "Name" and tag["Value"] == subnet_name
+            for tag in subnet["Tags"]
+        ):
+            return subnet["SubnetId"]
+
+    return None
+
+
+def check_ecr_repo_exists(repo_name: str) -> tuple[bool, dict]:
+    """
+    Checks if an ECR repository with the given name exists.
+
+    Args:
+        repo_name: The name of the ECR repository to check.
+
+    Returns:
+        True if the repository exists, False otherwise.
+    """
+    ecr_client = boto3.client("ecr")
+    try:
+        print("ecr repo_name to check:", repo_name)
+        response = ecr_client.describe_repositories(repositoryNames=[repo_name])
+        # If describe_repositories succeeds and returns a list of repositories,
+        # and the list is not empty, the repository exists.
+        return len(response["repositories"]) > 0, response["repositories"][0]
+    except ClientError as e:
+        # Check for the specific error code indicating the repository doesn't exist
+        if e.response["Error"]["Code"] == "RepositoryNotFoundException":
+            return False, {}
+        else:
+            # Re-raise other exceptions to handle unexpected errors
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, {}
+
+
+def check_codebuild_project_exists(
+    project_name: str,
+):  # Adjust return type hint as needed
+    """
+    Checks if a CodeBuild project with the given name exists.
+
+    Args:
+        project_name: The name of the CodeBuild project to check.
+
+    Returns:
+        A tuple:
+        - The first element is True if the project exists, False otherwise.
+        - The second element is the project object (dictionary) if found,
+          None otherwise.
+    """
+    codebuild_client = boto3.client("codebuild")
+    try:
+        # Use batch_get_projects with a list containing the single project name
+        response = codebuild_client.batch_get_projects(names=[project_name])
+
+        # The response for batch_get_projects includes 'projects' (found)
+        # and 'projectsNotFound' (not found).
+        if response["projects"]:
+            # If the project is found in the 'projects' list
+            print(f"CodeBuild project '{project_name}' found.")
+            return (
+                True,
+                response["projects"][0]["arn"],
+            )  # Return True and the project details dict
+        elif (
+            response["projectsNotFound"]
+            and project_name in response["projectsNotFound"]
+        ):
+            # If the project name is explicitly in the 'projectsNotFound' list
+            print(f"CodeBuild project '{project_name}' not found.")
+            return False, None
+        else:
+            # This case is less expected for a single name lookup,
+            # but could happen if there's an internal issue or the response
+            # structure is slightly different than expected for an error.
+            # It's safer to assume it wasn't found if not in 'projects'.
+            print(
+                f"CodeBuild project '{project_name}' not found (not in 'projects' list)."
+            )
+            return False, None
+
+    except ClientError as e:
+        # Catch specific ClientErrors. batch_get_projects might not throw
+        # 'InvalidInputException' for a non-existent project name if the
+        # name format is valid. It typically just lists it in projectsNotFound.
+        # However, other ClientErrors are possible (e.g., permissions).
+        print(
+            f"An AWS ClientError occurred checking CodeBuild project '{project_name}': {e}"
+        )
+        # Decide how to handle other ClientErrors - raising might be safer
+        raise  # Re-raise the original exception
+    except Exception as e:
+        print(
+            f"An unexpected non-ClientError occurred checking CodeBuild project '{project_name}': {e}"
+        )
+        # Decide how to handle other errors
+        raise  # Re-raise the original exception
+
+
+def get_vpc_id_by_name(vpc_name: str) -> Optional[str]:
+    """
+    Finds a VPC ID by its 'Name' tag.
+    """
+    ec2_client = boto3.client("ec2")
+    try:
+        response = ec2_client.describe_vpcs(
+            Filters=[{"Name": "tag:Name", "Values": [vpc_name]}]
+        )
+        if response and response["Vpcs"]:
+            vpc_id = response["Vpcs"][0]["VpcId"]
+            print(f"VPC '{vpc_name}' found with ID: {vpc_id}")
+
+            # In get_vpc_id_by_name, after finding VPC ID:
+
+            # Look for NAT Gateways in this VPC
+            ec2_client = boto3.client("ec2")
+            nat_gateways = []
+            try:
+                response = ec2_client.describe_nat_gateways(
+                    Filters=[
+                        {"Name": "vpc-id", "Values": [vpc_id]},
+                        # Optional: Add a tag filter if you consistently tag your NATs
+                        # {'Name': 'tag:Name', 'Values': [f"{prefix}-nat-gateway"]}
+                    ]
+                )
+                nat_gateways = response.get("NatGateways", [])
+            except Exception as e:
+                print(
+                    f"Warning: Could not describe NAT Gateways in VPC '{vpc_id}': {e}"
+                )
+                # Decide how to handle this error - proceed or raise?
+
+            # Decide how to identify the specific NAT Gateway you want to check for.
+
+            return vpc_id, nat_gateways
+        else:
+            print(f"VPC '{vpc_name}' not found.")
+            return None
+    except Exception as e:
+        print(f"An unexpected error occurred finding VPC '{vpc_name}': {e}")
+        raise
+
+
+# --- Helper to fetch all existing subnets in a VPC once ---
+def _get_existing_subnets_in_vpc(vpc_id: str) -> Dict[str, Any]:
+    """
+    Fetches all subnets in a given VPC.
+    Returns a dictionary with 'by_name' (map of name to subnet data),
+    'by_id' (map of id to subnet data), and 'cidr_networks' (list of ipaddress.IPv4Network).
+    """
+    ec2_client = boto3.client("ec2")
+    existing_subnets_data = {
+        "by_name": {},  # {subnet_name: {'id': 'subnet-id', 'cidr': 'x.x.x.x/x'}}
+        "by_id": {},  # {subnet_id: {'name': 'subnet-name', 'cidr': 'x.x.x.x/x'}}
+        "cidr_networks": [],  # List of ipaddress.IPv4Network objects
+    }
+    try:
+        response = ec2_client.describe_subnets(
+            Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
+        )
+        for s in response.get("Subnets", []):
+            subnet_id = s["SubnetId"]
+            cidr_block = s.get("CidrBlock")
+            # Extract 'Name' tag, which is crucial for lookup by name
+            name_tag = next(
+                (tag["Value"] for tag in s.get("Tags", []) if tag["Key"] == "Name"),
+                None,
+            )
+
+            subnet_info = {"id": subnet_id, "cidr": cidr_block, "name": name_tag}
+
+            if name_tag:
+                existing_subnets_data["by_name"][name_tag] = subnet_info
+            existing_subnets_data["by_id"][subnet_id] = subnet_info
+
+            if cidr_block:
+                try:
+                    existing_subnets_data["cidr_networks"].append(
+                        ipaddress.ip_network(cidr_block, strict=False)
+                    )
+                except ValueError:
+                    print(
+                        f"Warning: Existing subnet {subnet_id} has an invalid CIDR: {cidr_block}. Skipping for overlap check."
+                    )
+
+        print(
+            f"Fetched {len(response.get('Subnets', []))} existing subnets from VPC '{vpc_id}'."
+        )
+    except Exception as e:
+        print(
+            f"Error describing existing subnets in VPC '{vpc_id}': {e}. Cannot perform full validation."
+        )
+        raise  # Re-raise if this essential step fails
+
+    return existing_subnets_data
+
+
+# --- Modified validate_subnet_creation_parameters to take pre-fetched data ---
+def validate_subnet_creation_parameters(
+    vpc_id: str,
+    proposed_subnets_data: List[
+        Dict[str, str]
+    ],  # e.g., [{'name': 'my-public-subnet', 'cidr': '10.0.0.0/24', 'az': 'us-east-1a'}]
+    existing_aws_subnets_data: Dict[
+        str, Any
+    ],  # Pre-fetched data from _get_existing_subnets_in_vpc
+) -> None:
+    """
+    Validates proposed subnet names and CIDR blocks against existing AWS subnets
+    in the specified VPC and against each other.
+    This function uses pre-fetched AWS subnet data.
+
+    Args:
+        vpc_id: The ID of the VPC (for logging/error messages).
+        proposed_subnets_data: A list of dictionaries, where each dict represents
+                               a proposed subnet with 'name', 'cidr', and 'az'.
+        existing_aws_subnets_data: Dictionary containing existing AWS subnet data
+                                   (e.g., from _get_existing_subnets_in_vpc).
+
+    Raises:
+        ValueError: If any proposed subnet name or CIDR block
+                    conflicts with existing AWS resources or other proposed resources.
+    """
+    if not proposed_subnets_data:
+        print("No proposed subnet data provided for validation. Skipping.")
+        return
+
+    print(
+        f"--- Starting pre-synth validation for VPC '{vpc_id}' with proposed subnets ---"
+    )
+
+    print("Existing subnet data:", pd.DataFrame(existing_aws_subnets_data["by_name"]))
+
+    existing_aws_subnet_names = set(existing_aws_subnets_data["by_name"].keys())
+    existing_aws_cidr_networks = existing_aws_subnets_data["cidr_networks"]
+
+    # Sets to track names and list to track networks for internal batch consistency
+    proposed_names_seen: set[str] = set()
+    proposed_cidr_networks_seen: List[ipaddress.IPv4Network] = []
+
+    for i, proposed_subnet in enumerate(proposed_subnets_data):
+        subnet_name = proposed_subnet.get("name")
+        cidr_block_str = proposed_subnet.get("cidr")
+        availability_zone = proposed_subnet.get("az")
+
+        if not all([subnet_name, cidr_block_str, availability_zone]):
+            raise ValueError(
+                f"Proposed subnet at index {i} is incomplete. Requires 'name', 'cidr', and 'az'."
+            )
+
+        # 1. Check for duplicate names within the proposed batch
+        if subnet_name in proposed_names_seen:
+            raise ValueError(
+                f"Proposed subnet name '{subnet_name}' is duplicated within the input list."
+            )
+        proposed_names_seen.add(subnet_name)
+
+        # 2. Check for duplicate names against existing AWS subnets
+        if subnet_name in existing_aws_subnet_names:
+            print(
+                f"Proposed subnet name '{subnet_name}' already exists in VPC '{vpc_id}'."
+            )
+
+        # Parse proposed CIDR
+        try:
+            proposed_net = ipaddress.ip_network(cidr_block_str, strict=False)
+        except ValueError as e:
+            raise ValueError(
+                f"Invalid CIDR format '{cidr_block_str}' for proposed subnet '{subnet_name}': {e}"
+            )
+
+        # 3. Check for overlapping CIDRs within the proposed batch
+        for existing_proposed_net in proposed_cidr_networks_seen:
+            if proposed_net.overlaps(existing_proposed_net):
+                raise ValueError(
+                    f"Proposed CIDR '{cidr_block_str}' for subnet '{subnet_name}' "
+                    f"overlaps with another proposed CIDR '{str(existing_proposed_net)}' "
+                    f"within the same batch."
+                )
+
+        # 4. Check for overlapping CIDRs against existing AWS subnets
+        for existing_aws_net in existing_aws_cidr_networks:
+            if proposed_net.overlaps(existing_aws_net):
+                raise ValueError(
+                    f"Proposed CIDR '{cidr_block_str}' for subnet '{subnet_name}' "
+                    f"overlaps with an existing AWS subnet CIDR '{str(existing_aws_net)}' "
+                    f"in VPC '{vpc_id}'."
+                )
+
+        # If all checks pass for this subnet, add its network to the list for subsequent checks
+        proposed_cidr_networks_seen.append(proposed_net)
+        print(
+            f"Validation successful for proposed subnet '{subnet_name}' with CIDR '{cidr_block_str}'."
+        )
+
+    print(
+        f"--- All proposed subnets passed pre-synth validation checks for VPC '{vpc_id}'. ---"
+    )
+
+
+# --- Modified check_subnet_exists_by_name (Uses pre-fetched data) ---
+def check_subnet_exists_by_name(
+    subnet_name: str, existing_aws_subnets_data: Dict[str, Any]
+) -> Tuple[bool, Optional[str]]:
+    """
+    Checks if a subnet with the given name exists within the pre-fetched data.
+
+    Args:
+        subnet_name: The 'Name' tag value of the subnet to check.
+        existing_aws_subnets_data: Dictionary containing existing AWS subnet data
+                                   (e.g., from _get_existing_subnets_in_vpc).
+
+    Returns:
+        A tuple:
+        - The first element is True if the subnet exists, False otherwise.
+        - The second element is the Subnet ID if found, None otherwise.
+    """
+    subnet_info = existing_aws_subnets_data["by_name"].get(subnet_name)
+    if subnet_info:
+        print(f"Subnet '{subnet_name}' found with ID: {subnet_info['id']}")
+        return True, subnet_info["id"]
+    else:
+        print(f"Subnet '{subnet_name}' not found.")
+        return False, None
+
+
+def create_nat_gateway(
+    scope: Construct,
+    public_subnet_for_nat: ec2.ISubnet,  # Expects a proper ISubnet
+    nat_gateway_name: str,
+    nat_gateway_id_context_key: str,
+) -> str:
+    """
+    Creates a single NAT Gateway in the specified public subnet.
+    It does not handle lookup from context; the calling stack should do that.
+    Returns the CloudFormation Ref of the NAT Gateway ID.
+    """
+    print(
+        f"Defining a new NAT Gateway '{nat_gateway_name}' in subnet '{public_subnet_for_nat.subnet_id}'."
+    )
+
+    # Create an Elastic IP for the NAT Gateway
+    eip = ec2.CfnEIP(
+        scope,
+        NAT_GATEWAY_EIP_NAME,
+        tags=[CfnTag(key="Name", value=NAT_GATEWAY_EIP_NAME)],
+    )
+
+    # Create the NAT Gateway
+    nat_gateway_logical_id = nat_gateway_name.replace("-", "") + "NatGateway"
+    nat_gateway = ec2.CfnNatGateway(
+        scope,
+        nat_gateway_logical_id,
+        subnet_id=public_subnet_for_nat.subnet_id,  # Associate with the public subnet
+        allocation_id=eip.attr_allocation_id,  # Associate with the EIP
+        tags=[CfnTag(key="Name", value=nat_gateway_name)],
+    )
+    # The NAT GW depends on the EIP. The dependency on the subnet is implicit via subnet_id.
+    nat_gateway.add_dependency(eip)
+
+    # *** CRUCIAL: Use CfnOutput to export the ID after deployment ***
+    # This is how you will get the ID to put into cdk.context.json
+    CfnOutput(
+        scope,
+        "SingleNatGatewayIdOutput",
+        value=nat_gateway.ref,
+        description=f"Physical ID of the Single NAT Gateway. Add this to cdk.context.json under the key '{nat_gateway_id_context_key}'.",
+        export_name=f"{scope.stack_name}-NatGatewayId",  # Make export name unique
+    )
+
+    print(
+        f"CDK: Defined new NAT Gateway '{nat_gateway.ref}'. Its physical ID will be available in the stack outputs after deployment."
+    )
+    # Return the tokenised reference for use within this synthesis
+    return nat_gateway.ref
+
+
+def create_subnets(
+    scope: Construct,
+    vpc: ec2.IVpc,
+    prefix: str,
+    subnet_names: List[str],
+    cidr_blocks: List[str],
+    availability_zones: List[str],
+    is_public: bool,
+    internet_gateway_id: Optional[str] = None,
+    single_nat_gateway_id: Optional[str] = None,
+) -> Tuple[List[ec2.CfnSubnet], List[ec2.CfnRouteTable]]:
+    """
+    Creates subnets using L2 constructs but returns the underlying L1 Cfn objects
+    for backward compatibility.
+    """
+    # --- Validations remain the same ---
+    if not (len(subnet_names) == len(cidr_blocks) == len(availability_zones) > 0):
+        raise ValueError(
+            "Subnet names, CIDR blocks, and Availability Zones lists must be non-empty and match in length."
+        )
+    if is_public and not internet_gateway_id:
+        raise ValueError("internet_gateway_id must be provided for public subnets.")
+    if not is_public and not single_nat_gateway_id:
+        raise ValueError(
+            "single_nat_gateway_id must be provided for private subnets when using a single NAT Gateway."
+        )
+
+    # --- We will populate these lists with the L1 objects to return ---
+    created_subnets: List[ec2.CfnSubnet] = []
+    created_route_tables: List[ec2.CfnRouteTable] = []
+
+    subnet_type_tag = "public" if is_public else "private"
+
+    for i, subnet_name in enumerate(subnet_names):
+        logical_id = f"{prefix}{subnet_type_tag.capitalize()}Subnet{i+1}"
+
+        # 1. Create the L2 Subnet (this is the easy part)
+        subnet = ec2.Subnet(
+            scope,
+            logical_id,
+            vpc_id=vpc.vpc_id,
+            cidr_block=cidr_blocks[i],
+            availability_zone=availability_zones[i],
+            map_public_ip_on_launch=is_public,
+        )
+        Tags.of(subnet).add("Name", subnet_name)
+        Tags.of(subnet).add("Type", subnet_type_tag)
+
+        if is_public:
+            # The subnet's route_table is automatically created by the L2 Subnet construct
+            try:
+                subnet.add_route(
+                    "DefaultInternetRoute",  # A logical ID for the CfnRoute resource
+                    router_id=internet_gateway_id,
+                    router_type=ec2.RouterType.GATEWAY,
+                    # destination_cidr_block="0.0.0.0/0" is the default for this method
+                )
+            except Exception as e:
+                print("Could not create IGW route for public subnet due to:", e)
+            print(f"CDK: Defined public L2 subnet '{subnet_name}' and added IGW route.")
+        else:
+            try:
+                # Using .add_route() for private subnets as well for consistency
+                subnet.add_route(
+                    "DefaultNatRoute",  # A logical ID for the CfnRoute resource
+                    router_id=single_nat_gateway_id,
+                    router_type=ec2.RouterType.NAT_GATEWAY,
+                )
+            except Exception as e:
+                print("Could not create NAT gateway route for public subnet due to:", e)
+            print(
+                f"CDK: Defined private L2 subnet '{subnet_name}' and added NAT GW route."
+            )
+
+        route_table = subnet.route_table
+
+        created_subnets.append(subnet)
+        created_route_tables.append(route_table)
+
+    return created_subnets, created_route_tables
+
+
+def ingress_rule_exists(security_group: str, peer: str, port: str):
+    for rule in security_group.connections.security_groups:
+        if port:
+            if rule.peer == peer and rule.connection == port:
+                return True
+        else:
+            if rule.peer == peer:
+                return True
+    return False
+
+
+def check_for_existing_user_pool(user_pool_name: str):
+    cognito_client = boto3.client("cognito-idp")
+    list_pools_response = cognito_client.list_user_pools(
+        MaxResults=60
+    )  # MaxResults up to 60
+
+    # ListUserPools might require pagination if you have more than 60 pools
+    # This simple example doesn't handle pagination, which could miss your pool
+
+    existing_user_pool_id = ""
+
+    for pool in list_pools_response.get("UserPools", []):
+        if pool.get("Name") == user_pool_name:
+            existing_user_pool_id = pool["Id"]
+            print(
+                f"Found existing user pool by name '{user_pool_name}' with ID: {existing_user_pool_id}"
+            )
+            break  # Found the one we're looking for
+
+    if existing_user_pool_id:
+        return True, existing_user_pool_id, pool
+    else:
+        return False, "", ""
+
+
+def check_for_existing_user_pool_client(user_pool_id: str, user_pool_client_name: str):
+    """
+    Checks if a Cognito User Pool Client with the given name exists in the specified User Pool.
+
+    Args:
+        user_pool_id: The ID of the Cognito User Pool.
+        user_pool_client_name: The name of the User Pool Client to check for.
+
+    Returns:
+        A tuple:
+        - True, client_id, client_details if the client exists.
+        - False, "", {} otherwise.
+    """
+    cognito_client = boto3.client("cognito-idp")
+    next_token = "string"
+
+    while True:
+        try:
+            response = cognito_client.list_user_pool_clients(
+                UserPoolId=user_pool_id, MaxResults=60, NextToken=next_token
+            )
+        except cognito_client.exceptions.ResourceNotFoundException:
+            print(f"Error: User pool with ID '{user_pool_id}' not found.")
+            return False, "", {}
+
+        except cognito_client.exceptions.InvalidParameterException:
+            print(f"Error: No app clients for '{user_pool_id}' found.")
+            return False, "", {}
+
+        except Exception as e:
+            print("Could not check User Pool clients due to:", e)
+
+        for client in response.get("UserPoolClients", []):
+            if client.get("ClientName") == user_pool_client_name:
+                print(
+                    f"Found existing user pool client '{user_pool_client_name}' with ID: {client['ClientId']}"
+                )
+                return True, client["ClientId"], client
+
+        next_token = response.get("NextToken")
+        if not next_token:
+            break
+
+    return False, "", {}
+
+
+def check_for_secret(secret_name: str, secret_value: dict = ""):
+    """
+    Checks if a Secrets Manager secret with the given name exists.
+    If it doesn't exist, it creates the secret.
+
+    Args:
+        secret_name: The name of the Secrets Manager secret.
+        secret_value: A dictionary containing the key-value pairs for the secret.
+
+    Returns:
+        True if the secret existed or was created, False otherwise (due to other errors).
+    """
+    secretsmanager_client = boto3.client("secretsmanager")
+
+    try:
+        # Try to get the secret. If it doesn't exist, a ResourceNotFoundException will be raised.
+        secret_value = secretsmanager_client.get_secret_value(SecretId=secret_name)
+        print("Secret already exists.")
+        return True, secret_value
+    except secretsmanager_client.exceptions.ResourceNotFoundException:
+        print("Secret not found")
+        return False, {}
+    except Exception as e:
+        # Handle other potential exceptions during the get operation
+        print(f"Error checking for secret: {e}")
+        return False, {}
+
+
+def check_alb_exists(
+    load_balancer_name: str, region_name: str = None
+) -> tuple[bool, dict]:
+    """
+    Checks if an Application Load Balancer (ALB) with the given name exists.
+
+    Args:
+        load_balancer_name: The name of the ALB to check.
+        region_name: The AWS region to check in.  If None, uses the default
+                     session region.
+
+    Returns:
+        A tuple:
+        - The first element is True if the ALB exists, False otherwise.
+        - The second element is the ALB object (dictionary) if found,
+          None otherwise.  Specifically, it returns the first element of
+          the LoadBalancers list from the describe_load_balancers response.
+    """
+    if region_name:
+        elbv2_client = boto3.client("elbv2", region_name=region_name)
+    else:
+        elbv2_client = boto3.client("elbv2")
+    try:
+        response = elbv2_client.describe_load_balancers(Names=[load_balancer_name])
+        if response["LoadBalancers"]:
+            return (
+                True,
+                response["LoadBalancers"][0],
+            )  # Return True and the first ALB object
+        else:
+            return False, {}
+    except ClientError as e:
+        #  If the error indicates the ALB doesn't exist, return False
+        if e.response["Error"]["Code"] == "LoadBalancerNotFound":
+            return False, {}
+        else:
+            # Re-raise other exceptions
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, {}
+
+
+def check_fargate_task_definition_exists(
+    task_definition_name: str, region_name: str = None
+) -> tuple[bool, dict]:
+    """
+    Checks if a Fargate task definition with the given name exists.
+
+    Args:
+        task_definition_name: The name or ARN of the task definition to check.
+        region_name: The AWS region to check in. If None, uses the default
+                     session region.
+
+    Returns:
+        A tuple:
+        - The first element is True if the task definition exists, False otherwise.
+        - The second element is the task definition object (dictionary) if found,
+          None otherwise.  Specifically, it returns the first element of the
+          taskDefinitions list from the describe_task_definition response.
+    """
+    if region_name:
+        ecs_client = boto3.client("ecs", region_name=region_name)
+    else:
+        ecs_client = boto3.client("ecs")
+    try:
+        response = ecs_client.describe_task_definition(
+            taskDefinition=task_definition_name
+        )
+        # If describe_task_definition succeeds, it returns the task definition.
+        # We can directly return True and the task definition.
+        return True, response["taskDefinition"]
+    except ClientError as e:
+        # Check for the error code indicating the task definition doesn't exist.
+        if (
+            e.response["Error"]["Code"] == "ClientException"
+            and "Task definition" in e.response["Message"]
+            and "does not exist" in e.response["Message"]
+        ):
+            return False, {}
+        else:
+            # Re-raise other exceptions.
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, {}
+
+
+def check_ecs_service_exists(
+    cluster_name: str, service_name: str, region_name: str = None
+) -> tuple[bool, dict]:
+    """
+    Checks if an ECS service with the given name exists in the specified cluster.
+
+    Args:
+        cluster_name: The name or ARN of the ECS cluster.
+        service_name: The name of the ECS service to check.
+        region_name: The AWS region to check in. If None, uses the default
+                     session region.
+
+    Returns:
+        A tuple:
+        - The first element is True if the service exists, False otherwise.
+        - The second element is the service object (dictionary) if found,
+          None otherwise.
+    """
+    if region_name:
+        ecs_client = boto3.client("ecs", region_name=region_name)
+    else:
+        ecs_client = boto3.client("ecs")
+    try:
+        response = ecs_client.describe_services(
+            cluster=cluster_name, services=[service_name]
+        )
+        if response["services"]:
+            return (
+                True,
+                response["services"][0],
+            )  # Return True and the first service object
+        else:
+            return False, {}
+    except ClientError as e:
+        # Check for the error code indicating the service doesn't exist.
+        if e.response["Error"]["Code"] == "ClusterNotFoundException":
+            return False, {}
+        elif e.response["Error"]["Code"] == "ServiceNotFoundException":
+            return False, {}
+        else:
+            # Re-raise other exceptions.
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, {}
+
+
+def check_cloudfront_distribution_exists(
+    distribution_name: str, region_name: str = None
+) -> tuple[bool, dict | None]:
+    """
+    Checks if a CloudFront distribution with the given name exists.
+
+    Args:
+        distribution_name: The name of the CloudFront distribution to check.
+        region_name: The AWS region to check in. If None, uses the default
+                     session region.  Note: CloudFront is a global service,
+                     so the region is usually 'us-east-1', but this parameter
+                     is included for completeness.
+
+    Returns:
+        A tuple:
+        - The first element is True if the distribution exists, False otherwise.
+        - The second element is the distribution object (dictionary) if found,
+          None otherwise.  Specifically, it returns the first element of the
+          DistributionList from the ListDistributions response.
+    """
+    if region_name:
+        cf_client = boto3.client("cloudfront", region_name=region_name)
+    else:
+        cf_client = boto3.client("cloudfront")
+    try:
+        response = cf_client.list_distributions()
+        if "Items" in response["DistributionList"]:
+            for distribution in response["DistributionList"]["Items"]:
+                # CloudFront doesn't directly filter by name, so we have to iterate.
+                if (
+                    distribution["AliasSet"]["Items"]
+                    and distribution["AliasSet"]["Items"][0] == distribution_name
+                ):
+                    return True, distribution
+            return False, None
+        else:
+            return False, None
+    except ClientError as e:
+        #  If the error indicates the Distribution doesn't exist, return False
+        if e.response["Error"]["Code"] == "NoSuchDistribution":
+            return False, None
+        else:
+            # Re-raise other exceptions
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, None
+
+
+def create_web_acl_with_common_rules(
+    scope: Construct, web_acl_name: str, waf_scope: str = "CLOUDFRONT"
+):
+    """
+    Use CDK to create a web ACL based on an AWS common rule set with overrides.
+    This function now expects a 'scope' argument, typically 'self' from your stack,
+    as CfnWebACL requires a construct scope.
+    """
+
+    # Create full list of rules
+    rules = []
+    aws_ruleset_names = [
+        "AWSManagedRulesCommonRuleSet",
+        "AWSManagedRulesKnownBadInputsRuleSet",
+        "AWSManagedRulesAmazonIpReputationList",
+    ]
+
+    # Use a separate counter to assign unique priorities sequentially
+    priority_counter = 1
+
+    for aws_rule_name in aws_ruleset_names:
+        current_rule_action_overrides = None
+
+        # All managed rule groups need an override_action.
+        # 'none' means use the managed rule group's default action.
+        current_override_action = wafv2.CfnWebACL.OverrideActionProperty(none={})
+
+        current_priority = priority_counter
+        priority_counter += 1
+
+        if aws_rule_name == "AWSManagedRulesCommonRuleSet":
+            current_rule_action_overrides = [
+                wafv2.CfnWebACL.RuleActionOverrideProperty(
+                    name="SizeRestrictions_BODY",
+                    action_to_use=wafv2.CfnWebACL.RuleActionProperty(allow={}),
+                )
+            ]
+            # No need to set current_override_action here, it's already set above.
+            # If you wanted this specific rule to have a *fixed* priority, you'd handle it differently
+            # For now, it will get priority 1 from the counter.
+
+        rule_property = wafv2.CfnWebACL.RuleProperty(
+            name=aws_rule_name,
+            priority=current_priority,
+            statement=wafv2.CfnWebACL.StatementProperty(
+                managed_rule_group_statement=wafv2.CfnWebACL.ManagedRuleGroupStatementProperty(
+                    vendor_name="AWS",
+                    name=aws_rule_name,
+                    rule_action_overrides=current_rule_action_overrides,
+                )
+            ),
+            visibility_config=wafv2.CfnWebACL.VisibilityConfigProperty(
+                cloud_watch_metrics_enabled=True,
+                metric_name=aws_rule_name,
+                sampled_requests_enabled=True,
+            ),
+            override_action=current_override_action,  # THIS IS THE CRUCIAL PART FOR ALL MANAGED RULES
+        )
+
+        rules.append(rule_property)
+
+    # Add the rate limit rule
+    rate_limit_priority = priority_counter  # Use the next available priority
+    rules.append(
+        wafv2.CfnWebACL.RuleProperty(
+            name="RateLimitRule",
+            priority=rate_limit_priority,
+            statement=wafv2.CfnWebACL.StatementProperty(
+                rate_based_statement=wafv2.CfnWebACL.RateBasedStatementProperty(
+                    limit=1000, aggregate_key_type="IP"
+                )
+            ),
+            visibility_config=wafv2.CfnWebACL.VisibilityConfigProperty(
+                cloud_watch_metrics_enabled=True,
+                metric_name="RateLimitRule",
+                sampled_requests_enabled=True,
+            ),
+            action=wafv2.CfnWebACL.RuleActionProperty(block={}),
+        )
+    )
+
+    web_acl = wafv2.CfnWebACL(
+        scope,
+        "WebACL",
+        name=web_acl_name,
+        default_action=wafv2.CfnWebACL.DefaultActionProperty(allow={}),
+        scope=waf_scope,
+        visibility_config=wafv2.CfnWebACL.VisibilityConfigProperty(
+            cloud_watch_metrics_enabled=True,
+            metric_name="webACL",
+            sampled_requests_enabled=True,
+        ),
+        rules=rules,
+    )
+
+    CfnOutput(scope, "WebACLArn", value=web_acl.attr_arn)
+
+    return web_acl
+
+
+def check_web_acl_exists(
+    web_acl_name: str, scope: str, region_name: str = None
+) -> tuple[bool, dict]:
+    """
+    Checks if a Web ACL with the given name and scope exists.
+
+    Args:
+        web_acl_name: The name of the Web ACL to check.
+        scope: The scope of the Web ACL ('CLOUDFRONT' or 'REGIONAL').
+        region_name: The AWS region to check in. Required for REGIONAL scope.
+                     If None, uses the default session region.  For CLOUDFRONT,
+                     the region should be 'us-east-1'.
+
+    Returns:
+        A tuple:
+        - The first element is True if the Web ACL exists, False otherwise.
+        - The second element is the Web ACL object (dictionary) if found,
+          None otherwise.
+    """
+    if scope not in ["CLOUDFRONT", "REGIONAL"]:
+        raise ValueError("Scope must be either 'CLOUDFRONT' or 'REGIONAL'")
+
+    if scope == "REGIONAL" and not region_name:
+        raise ValueError("Region name is required for REGIONAL scope")
+
+    if scope == "CLOUDFRONT":
+        region_name = "us-east-1"  # CloudFront scope requires us-east-1
+
+    if region_name:
+        waf_client = boto3.client("wafv2", region_name=region_name)
+    else:
+        waf_client = boto3.client("wafv2")
+    try:
+        response = waf_client.list_web_acls(Scope=scope)
+        if "WebACLs" in response:
+            for web_acl in response["WebACLs"]:
+                if web_acl["Name"] == web_acl_name:
+                    # Describe the Web ACL to get the full object.
+                    describe_response = waf_client.describe_web_acl(
+                        Name=web_acl_name, Scope=scope
+                    )
+                    return True, describe_response["WebACL"]
+            return False, {}
+        else:
+            return False, {}
+    except ClientError as e:
+        # Check for the error code indicating the web ACL doesn't exist.
+        if e.response["Error"]["Code"] == "ResourceNotFoundException":
+            return False, {}
+        else:
+            # Re-raise other exceptions.
+            raise
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        return False, {}
+
+
+def add_alb_https_listener_with_cert(
+    scope: Construct,
+    logical_id: str,  # A unique ID for this listener construct
+    alb: elb.ApplicationLoadBalancer,
+    acm_certificate_arn: Optional[
+        str
+    ],  # Optional: If None, no HTTPS listener will be created
+    default_target_group: elb.ITargetGroup,  # Mandatory: The target group to forward traffic to
+    listener_port_https: int = 443,
+    listener_open_to_internet: bool = False,  # Be cautious with True, ensure ALB security group restricts access
+    # --- Cognito Authentication Parameters ---
+    enable_cognito_auth: bool = False,
+    cognito_user_pool: Optional[cognito.IUserPool] = None,
+    cognito_user_pool_client: Optional[cognito.IUserPoolClient] = None,
+    cognito_user_pool_domain: Optional[
+        str
+    ] = None,  # E.g., "my-app-domain" for "my-app-domain.auth.region.amazoncognito.com"
+    cognito_auth_scope: Optional[
+        str
+    ] = "openid profile email",  # Default recommended scope
+    cognito_auth_on_unauthenticated_request: elb.UnauthenticatedAction = elb.UnauthenticatedAction.AUTHENTICATE,
+    stickiness_cookie_duration=None,
+    # --- End Cognito Parameters ---
+) -> Optional[elb.ApplicationListener]:
+    """
+    Conditionally adds an HTTPS listener to an ALB with an ACM certificate,
+    and optionally enables Cognito User Pool authentication.
+
+    Args:
+        scope (Construct): The scope in which to define this construct (e.g., your CDK Stack).
+        logical_id (str): A unique logical ID for the listener construct within the stack.
+        alb (elb.ApplicationLoadBalancer): The Application Load Balancer to add the listener to.
+        acm_certificate_arn (Optional[str]): The ARN of the ACM certificate to attach.
+                                             If None, the HTTPS listener will NOT be created.
+        default_target_group (elb.ITargetGroup): The default target group for the listener to forward traffic to.
+                                                 This is mandatory for a functional listener.
+        listener_port_https (int): The HTTPS port to listen on (default: 443).
+        listener_open_to_internet (bool): Whether the listener should allow connections from all sources.
+                                          If False (recommended), ensure your ALB's security group allows
+                                          inbound traffic on this port from desired sources.
+        enable_cognito_auth (bool): Set to True to enable Cognito User Pool authentication.
+        cognito_user_pool (Optional[cognito.IUserPool]): The Cognito User Pool object. Required if enable_cognito_auth is True.
+        cognito_user_pool_client (Optional[cognito.IUserPoolClient]): The Cognito User Pool App Client object. Required if enable_cognito_auth is True.
+        cognito_user_pool_domain (Optional[str]): The domain prefix for your Cognito User Pool. Required if enable_cognito_auth is True.
+        cognito_auth_scope (Optional[str]): The scope for the Cognito authentication.
+        cognito_auth_on_unauthenticated_request (elb.UnauthenticatedAction): Action for unauthenticated requests.
+                                                                           Defaults to AUTHENTICATE (redirect to login).
+
+    Returns:
+        Optional[elb.ApplicationListener]: The created ApplicationListener if successful,
+                                           None if no ACM certificate ARN was provided.
+    """
+    https_listener = None
+    if acm_certificate_arn:
+        certificates_list = [elb.ListenerCertificate.from_arn(acm_certificate_arn)]
+        print(
+            f"Attempting to add ALB HTTPS listener on port {listener_port_https} with ACM certificate: {acm_certificate_arn}"
+        )
+
+        # Determine the default action based on whether Cognito auth is enabled
+        default_action = None
+        if enable_cognito_auth is True:
+            if not all(
+                [cognito_user_pool, cognito_user_pool_client, cognito_user_pool_domain]
+            ):
+                raise ValueError(
+                    "Cognito User Pool, Client, and Domain must be provided if enable_cognito_auth is True."
+                )
+            print(
+                f"Enabling Cognito authentication with User Pool: {cognito_user_pool.user_pool_id}"
+            )
+
+            default_action = elb_act.AuthenticateCognitoAction(
+                next=elb.ListenerAction.forward(
+                    [default_target_group]
+                ),  # After successful auth, forward to TG
+                user_pool=cognito_user_pool,
+                user_pool_client=cognito_user_pool_client,
+                user_pool_domain=cognito_user_pool_domain,
+                scope=cognito_auth_scope,
+                on_unauthenticated_request=cognito_auth_on_unauthenticated_request,
+                session_timeout=stickiness_cookie_duration,
+                # Additional options you might want to configure:
+                # session_cookie_name="AWSELBCookies"
+            )
+        else:
+            default_action = elb.ListenerAction.forward([default_target_group])
+            print("Cognito authentication is NOT enabled for this listener.")
+
+        # Add the HTTPS listener
+        https_listener = alb.add_listener(
+            logical_id,
+            port=listener_port_https,
+            open=listener_open_to_internet,
+            certificates=certificates_list,
+            default_action=default_action,  # Use the determined default action
+        )
+        print(f"ALB HTTPS listener on port {listener_port_https} defined.")
+    else:
+        print("ACM_CERTIFICATE_ARN is not provided. Skipping HTTPS listener creation.")
+
+    return https_listener
+
+
+def ensure_folder_exists(output_folder: str):
+    """Checks if the specified folder exists, creates it if not."""
+
+    if not os.path.exists(output_folder):
+        # Create the folder if it doesn't exist
+        os.makedirs(output_folder, exist_ok=True)
+        print(f"Created the {output_folder} folder.")
+    else:
+        print(f"The {output_folder} folder already exists.")
+
+
+def create_basic_config_env(
+    out_dir: str = "config",
+    S3_LOG_CONFIG_BUCKET_NAME=S3_LOG_CONFIG_BUCKET_NAME,
+    S3_OUTPUT_BUCKET_NAME=S3_OUTPUT_BUCKET_NAME,
+    ACCESS_LOG_DYNAMODB_TABLE_NAME=ACCESS_LOG_DYNAMODB_TABLE_NAME,
+    FEEDBACK_LOG_DYNAMODB_TABLE_NAME=FEEDBACK_LOG_DYNAMODB_TABLE_NAME,
+    USAGE_LOG_DYNAMODB_TABLE_NAME=USAGE_LOG_DYNAMODB_TABLE_NAME,
+):
+    """
+    Create a basic config.env file for the user to use with their newly deployed redaction app.
+    """
+    variables = {
+        "COGNITO_AUTH": "True",
+        "RUN_AWS_FUNCTIONS": "True",
+        "DISPLAY_FILE_NAMES_IN_LOGS": "False",
+        "SESSION_OUTPUT_FOLDER": "True",
+        "SAVE_LOGS_TO_DYNAMODB": "True",
+        "SHOW_COSTS": "True",
+        "SHOW_WHOLE_DOCUMENT_TEXTRACT_CALL_OPTIONS": "True",
+        "LOAD_PREVIOUS_TEXTRACT_JOBS_S3": "True",
+        "DOCUMENT_REDACTION_BUCKET": S3_LOG_CONFIG_BUCKET_NAME,
+        "TEXTRACT_WHOLE_DOCUMENT_ANALYSIS_BUCKET": S3_OUTPUT_BUCKET_NAME,
+        "ACCESS_LOG_DYNAMODB_TABLE_NAME": ACCESS_LOG_DYNAMODB_TABLE_NAME,
+        "FEEDBACK_LOG_DYNAMODB_TABLE_NAME": FEEDBACK_LOG_DYNAMODB_TABLE_NAME,
+        "USAGE_LOG_DYNAMODB_TABLE_NAME": USAGE_LOG_DYNAMODB_TABLE_NAME,
+    }
+
+    # Write variables to .env file
+    ensure_folder_exists(out_dir + "/")
+    env_file_path = os.path.abspath(os.path.join(out_dir, "config.env"))
+
+    # It's good practice to ensure the file exists before calling set_key repeatedly.
+    # set_key will create it, but for a loop, it might be cleaner to ensure it's empty/exists once.
+    if not os.path.exists(env_file_path):
+        with open(env_file_path, "w"):
+            pass  # Create empty file
+
+    for key, value in variables.items():
+        set_key(env_file_path, key, str(value), quote_mode="never")
+
+    return variables
+
+
+def start_codebuild_build(PROJECT_NAME: str, AWS_REGION: str = AWS_REGION):
+    """
+    Start an existing Codebuild project build
+    """
+
+    # --- Initialize CodeBuild client ---
+    client = boto3.client("codebuild", region_name=AWS_REGION)
+
+    try:
+        print(f"Attempting to start build for project: {PROJECT_NAME}")
+
+        response = client.start_build(projectName=PROJECT_NAME)
+
+        build_id = response["build"]["id"]
+        print(f"Successfully started build with ID: {build_id}")
+        print(f"Build ARN: {response['build']['arn']}")
+        print("Build URL (approximate - construct based on region and ID):")
+        print(
+            f"https://{AWS_REGION}.console.aws.amazon.com/codesuite/codebuild/projects/{PROJECT_NAME}/build/{build_id.split(':')[-1]}/detail"
+        )
+
+        # You can inspect the full response if needed
+        # print("\nFull response:")
+        # import json
+        # print(json.dumps(response, indent=2))
+
+    except client.exceptions.ResourceNotFoundException:
+        print(f"Error: Project '{PROJECT_NAME}' not found in region '{AWS_REGION}'.")
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+
+
+def upload_file_to_s3(
+    local_file_paths: List[str],
+    s3_key: str,
+    s3_bucket: str,
+    RUN_AWS_FUNCTIONS: str = "1",
+):
+    """
+    Uploads a file from local machine to Amazon S3.
+
+    Args:
+    - local_file_path: Local file path(s) of the file(s) to upload.
+    - s3_key: Key (path) to the file in the S3 bucket.
+    - s3_bucket: Name of the S3 bucket.
+
+    Returns:
+    - Message as variable/printed to console
+    """
+    final_out_message = []
+    final_out_message_str = ""
+
+    if RUN_AWS_FUNCTIONS == "1":
+        try:
+            if s3_bucket and local_file_paths:
+
+                s3_client = boto3.client("s3", region_name=AWS_REGION)
+
+                if isinstance(local_file_paths, str):
+                    local_file_paths = [local_file_paths]
+
+                for file in local_file_paths:
+                    if s3_client:
+                        # print(s3_client)
+                        try:
+                            # Get file name off file path
+                            file_name = os.path.basename(file)
+
+                            s3_key_full = s3_key + file_name
+                            print("S3 key: ", s3_key_full)
+
+                            s3_client.upload_file(file, s3_bucket, s3_key_full)
+                            out_message = (
+                                "File " + file_name + " uploaded successfully!"
+                            )
+                            print(out_message)
+
+                        except Exception as e:
+                            out_message = f"Error uploading file(s): {e}"
+                            print(out_message)
+
+                        final_out_message.append(out_message)
+                        final_out_message_str = "\n".join(final_out_message)
+
+                    else:
+                        final_out_message_str = "Could not connect to AWS."
+            else:
+                final_out_message_str = (
+                    "At least one essential variable is empty, could not upload to S3"
+                )
+        except Exception as e:
+            final_out_message_str = "Could not upload files to S3 due to: " + str(e)
+            print(final_out_message_str)
+    else:
+        final_out_message_str = "App not set to run AWS functions"
+
+    return final_out_message_str
+
+
+# Initialize ECS client
+def start_ecs_task(cluster_name, service_name):
+    ecs_client = boto3.client("ecs")
+
+    try:
+        # Update the service to set the desired count to 1
+        ecs_client.update_service(
+            cluster=cluster_name, service=service_name, desiredCount=1
+        )
+        return {
+            "statusCode": 200,
+            "body": f"Service {service_name} in cluster {cluster_name} has been updated to 1 task.",
+        }
+    except Exception as e:
+        return {"statusCode": 500, "body": f"Error updating service: {str(e)}"}

cdk/cdk_stack.py

@@ -0,0 +1,1869 @@
+import json  # You might still need json if loading task_definition.json
+import os
+from typing import Any, Dict, List
+
+from aws_cdk import (
+    CfnOutput,  # <-- Import CfnOutput directly
+    Duration,
+    RemovalPolicy,
+    SecretValue,
+    Stack,
+)
+from aws_cdk import aws_cloudfront as cloudfront
+from aws_cdk import aws_cloudfront_origins as origins
+from aws_cdk import aws_codebuild as codebuild
+from aws_cdk import aws_cognito as cognito
+from aws_cdk import aws_dynamodb as dynamodb  # Import the DynamoDB module
+from aws_cdk import aws_ec2 as ec2
+from aws_cdk import aws_ecr as ecr
+from aws_cdk import aws_ecs as ecs
+from aws_cdk import aws_elasticloadbalancingv2 as elbv2
+from aws_cdk import aws_iam as iam
+from aws_cdk import aws_kms as kms
+from aws_cdk import aws_logs as logs
+from aws_cdk import aws_s3 as s3
+from aws_cdk import aws_secretsmanager as secretsmanager
+from aws_cdk import aws_wafv2 as wafv2
+from cdk_config import (
+    ACCESS_LOG_DYNAMODB_TABLE_NAME,
+    ACM_SSL_CERTIFICATE_ARN,
+    ALB_NAME,
+    ALB_NAME_SECURITY_GROUP_NAME,
+    ALB_TARGET_GROUP_NAME,
+    AWS_ACCOUNT_ID,
+    AWS_MANAGED_TASK_ROLES_LIST,
+    AWS_REGION,
+    CDK_PREFIX,
+    CLOUDFRONT_DISTRIBUTION_NAME,
+    CLOUDFRONT_GEO_RESTRICTION,
+    CLUSTER_NAME,
+    CODEBUILD_PROJECT_NAME,
+    CODEBUILD_ROLE_NAME,
+    COGNITO_ACCESS_TOKEN_VALIDITY,
+    COGNITO_ID_TOKEN_VALIDITY,
+    COGNITO_REDIRECTION_URL,
+    COGNITO_REFRESH_TOKEN_VALIDITY,
+    COGNITO_USER_POOL_CLIENT_NAME,
+    COGNITO_USER_POOL_CLIENT_SECRET_NAME,
+    COGNITO_USER_POOL_DOMAIN_PREFIX,
+    COGNITO_USER_POOL_NAME,
+    CUSTOM_HEADER,
+    CUSTOM_HEADER_VALUE,
+    CUSTOM_KMS_KEY_NAME,
+    DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS,
+    ECR_CDK_REPO_NAME,
+    ECS_LOG_GROUP_NAME,
+    ECS_READ_ONLY_FILE_SYSTEM,
+    ECS_SECURITY_GROUP_NAME,
+    ECS_SERVICE_NAME,
+    ECS_TASK_CPU_SIZE,
+    ECS_TASK_EXECUTION_ROLE_NAME,
+    ECS_TASK_MEMORY_SIZE,
+    ECS_TASK_ROLE_NAME,
+    ECS_USE_FARGATE_SPOT,
+    EXISTING_IGW_ID,
+    FARGATE_TASK_DEFINITION_NAME,
+    FEEDBACK_LOG_DYNAMODB_TABLE_NAME,
+    GITHUB_REPO_BRANCH,
+    GITHUB_REPO_NAME,
+    GITHUB_REPO_USERNAME,
+    GRADIO_SERVER_PORT,
+    LOAD_BALANCER_WEB_ACL_NAME,
+    NAT_GATEWAY_NAME,
+    NEW_VPC_CIDR,
+    NEW_VPC_DEFAULT_NAME,
+    PRIVATE_SUBNET_AVAILABILITY_ZONES,
+    PRIVATE_SUBNET_CIDR_BLOCKS,
+    PRIVATE_SUBNETS_TO_USE,
+    PUBLIC_SUBNET_AVAILABILITY_ZONES,
+    PUBLIC_SUBNET_CIDR_BLOCKS,
+    PUBLIC_SUBNETS_TO_USE,
+    S3_LOG_CONFIG_BUCKET_NAME,
+    S3_OUTPUT_BUCKET_NAME,
+    SAVE_LOGS_TO_DYNAMODB,
+    SINGLE_NAT_GATEWAY_ID,
+    TASK_DEFINITION_FILE_LOCATION,
+    USAGE_LOG_DYNAMODB_TABLE_NAME,
+    USE_CLOUDFRONT,
+    USE_CUSTOM_KMS_KEY,
+    VPC_NAME,
+    WEB_ACL_NAME,
+)
+from cdk_functions import (  # Only keep CDK-native functions
+    add_alb_https_listener_with_cert,
+    add_custom_policies,
+    create_nat_gateway,
+    create_subnets,
+    create_web_acl_with_common_rules,
+)
+from constructs import Construct
+
+
+def _get_env_list(env_var_name: str) -> List[str]:
+    """Parses a comma-separated environment variable into a list of strings."""
+    value = env_var_name[1:-1].strip().replace('"', "").replace("'", "")
+    if not value:
+        return []
+    # Split by comma and filter out any empty strings that might result from extra commas
+    return [s.strip() for s in value.split(",") if s.strip()]
+
+
+# 1. Try to load CIDR/AZs from environment variables
+if PUBLIC_SUBNETS_TO_USE:
+    PUBLIC_SUBNETS_TO_USE = _get_env_list(PUBLIC_SUBNETS_TO_USE)
+if PRIVATE_SUBNETS_TO_USE:
+    PRIVATE_SUBNETS_TO_USE = _get_env_list(PRIVATE_SUBNETS_TO_USE)
+
+if PUBLIC_SUBNET_CIDR_BLOCKS:
+    PUBLIC_SUBNET_CIDR_BLOCKS = _get_env_list("PUBLIC_SUBNET_CIDR_BLOCKS")
+if PUBLIC_SUBNET_AVAILABILITY_ZONES:
+    PUBLIC_SUBNET_AVAILABILITY_ZONES = _get_env_list("PUBLIC_SUBNET_AVAILABILITY_ZONES")
+if PRIVATE_SUBNET_CIDR_BLOCKS:
+    PRIVATE_SUBNET_CIDR_BLOCKS = _get_env_list("PRIVATE_SUBNET_CIDR_BLOCKS")
+if PRIVATE_SUBNET_AVAILABILITY_ZONES:
+    PRIVATE_SUBNET_AVAILABILITY_ZONES = _get_env_list(
+        "PRIVATE_SUBNET_AVAILABILITY_ZONES"
+    )
+
+if AWS_MANAGED_TASK_ROLES_LIST:
+    AWS_MANAGED_TASK_ROLES_LIST = _get_env_list(AWS_MANAGED_TASK_ROLES_LIST)
+
+
+class CdkStack(Stack):
+
+    def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
+        super().__init__(scope, construct_id, **kwargs)
+
+        # --- Helper to get context values ---
+        def get_context_bool(key: str, default: bool = False) -> bool:
+            return self.node.try_get_context(key) or default
+
+        def get_context_str(key: str, default: str = None) -> str:
+            return self.node.try_get_context(key) or default
+
+        def get_context_dict(key: str, default: dict = None) -> dict:
+            return self.node.try_get_context(key) or default
+
+        def get_context_list_of_dicts(key: str) -> List[Dict[str, Any]]:
+            ctx_value = self.node.try_get_context(key)
+            if not isinstance(ctx_value, list):
+                print(
+                    f"Warning: Context key '{key}' not found or not a list. Returning empty list."
+                )
+                return []
+            # Optional: Add validation that all items in the list are dicts
+            return ctx_value
+
+        self.template_options.description = "Deployment of the 'doc_redaction' PDF, image, and XLSX/CSV redaction app. Git repo available at: https://github.com/seanpedrick-case/doc_redaction."
+
+        # --- VPC and Subnets (Assuming VPC is always lookup, Subnets are created/returned by create_subnets) ---
+        new_vpc_created = False
+        if VPC_NAME:
+            print("Looking for current VPC:", VPC_NAME)
+            try:
+                vpc = ec2.Vpc.from_lookup(self, "VPC", vpc_name=VPC_NAME)
+                print("Successfully looked up VPC:", vpc.vpc_id)
+            except Exception as e:
+                raise Exception(
+                    f"Could not look up VPC with name '{VPC_NAME}' due to: {e}"
+                )
+
+        elif NEW_VPC_DEFAULT_NAME:
+            new_vpc_created = True
+            print(
+                f"NEW_VPC_DEFAULT_NAME ('{NEW_VPC_DEFAULT_NAME}') is set. Creating a new VPC."
+            )
+
+            # Configuration for the new VPC
+            # You can make these configurable via context as well, e.g.,
+            # new_vpc_cidr = self.node.try_get_context("new_vpc_cidr") or "10.0.0.0/24"
+            # new_vpc_max_azs = self.node.try_get_context("new_vpc_max_azs") or 2 # Use 2 AZs by default for HA
+            # new_vpc_nat_gateways = self.node.try_get_context("new_vpc_nat_gateways") or new_vpc_max_azs # One NAT GW per AZ for HA
+            # or 1 for cost savings if acceptable
+            if not NEW_VPC_CIDR:
+                raise Exception(
+                    "App has been instructed to create a new VPC but not VPC CDR range provided to variable NEW_VPC_CIDR"
+                )
+
+            print("Provided NEW_VPC_CIDR range:", NEW_VPC_CIDR)
+
+            new_vpc_cidr = NEW_VPC_CIDR
+            new_vpc_max_azs = 2  # Creates resources in 2 AZs. Adjust as needed.
+
+            # For "a NAT gateway", you can set nat_gateways=1.
+            # For resilience (NAT GW per AZ), set nat_gateways=new_vpc_max_azs.
+            # The Vpc construct will create NAT Gateway(s) if subnet_type PRIVATE_WITH_EGRESS is used
+            # and nat_gateways > 0.
+            new_vpc_nat_gateways = (
+                1  # Creates a single NAT Gateway for cost-effectiveness.
+            )
+            # If you need one per AZ for higher availability, set this to new_vpc_max_azs.
+
+            vpc = ec2.Vpc(
+                self,
+                "MyNewLogicalVpc",  # This is the CDK construct ID
+                vpc_name=NEW_VPC_DEFAULT_NAME,
+                ip_addresses=ec2.IpAddresses.cidr(new_vpc_cidr),
+                max_azs=new_vpc_max_azs,
+                nat_gateways=new_vpc_nat_gateways,  # Number of NAT gateways to create
+                subnet_configuration=[
+                    ec2.SubnetConfiguration(
+                        name="Public",  # Name prefix for public subnets
+                        subnet_type=ec2.SubnetType.PUBLIC,
+                        cidr_mask=28,  # Adjust CIDR mask as needed (e.g., /24 provides ~250 IPs per subnet)
+                    ),
+                    ec2.SubnetConfiguration(
+                        name="Private",  # Name prefix for private subnets
+                        subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS,  # Ensures these subnets have NAT Gateway access
+                        cidr_mask=28,  # Adjust CIDR mask as needed
+                    ),
+                    # You could also add ec2.SubnetType.PRIVATE_ISOLATED if needed
+                ],
+                # Internet Gateway is created and configured automatically for PUBLIC subnets.
+                # Route tables for public subnets will point to the IGW.
+                # Route tables for PRIVATE_WITH_EGRESS subnets will point to the NAT Gateway(s).
+            )
+            print(
+                f"Successfully created new VPC: {vpc.vpc_id} with name '{NEW_VPC_DEFAULT_NAME}'"
+            )
+            # If nat_gateways > 0, vpc.nat_gateway_ips will contain EIPs if Vpc created them.
+            # vpc.public_subnets, vpc.private_subnets, vpc.isolated_subnets are populated.
+
+        else:
+            raise Exception(
+                "VPC_NAME for current VPC not found, and NEW_VPC_DEFAULT_NAME not found to create a new VPC"
+            )
+
+        # --- Subnet Handling (Check Context and Create/Import) ---
+        # Initialize lists to hold ISubnet objects (L2) and CfnSubnet/CfnRouteTable (L1)
+        # We will store ISubnet for consistency, as CfnSubnet has a .subnet_id property
+        self.public_subnets: List[ec2.ISubnet] = []
+        self.private_subnets: List[ec2.ISubnet] = []
+        # Store L1 CfnRouteTables explicitly if you need to reference them later
+        self.private_route_tables_cfn: List[ec2.CfnRouteTable] = []
+        self.public_route_tables_cfn: List[ec2.CfnRouteTable] = (
+            []
+        )  # New: to store public RTs
+
+        names_to_create_private = []
+        names_to_create_public = []
+
+        if not PUBLIC_SUBNETS_TO_USE and not PRIVATE_SUBNETS_TO_USE:
+            print(
+                "Warning: No public or private subnets specified in *_SUBNETS_TO_USE. Attempting to select from existing VPC subnets."
+            )
+
+            print("vpc.public_subnets:", vpc.public_subnets)
+            print("vpc.private_subnets:", vpc.private_subnets)
+
+            if (
+                vpc.public_subnets
+            ):  # These are already one_per_az if max_azs was used and Vpc created them
+                self.public_subnets.extend(vpc.public_subnets)
+            else:
+                self.node.add_warning("No public subnets found in the VPC.")
+
+            # Get private subnets with egress specifically
+            # selected_private_subnets_with_egress = vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS)
+
+            print(
+                f"Selected from VPC: {len(self.public_subnets)} public, {len(self.private_subnets)} private_with_egress subnets."
+            )
+
+            if (
+                len(self.public_subnets) < 1 or len(self.private_subnets) < 1
+            ):  # Simplified check for new VPC
+                # If new_vpc_max_azs was 1, you'd have 1 of each. If 2, then 2 of each.
+                # The original check ' < 2' might be too strict if new_vpc_max_azs=1
+                pass  # For new VPC, allow single AZ setups if configured that way. The VPC construct ensures one per AZ up to max_azs.
+
+            if not self.public_subnets and not self.private_subnets:
+                print(
+                    "Error: No public or private subnets could be found in the VPC for automatic selection. "
+                    "You must either specify subnets in *_SUBNETS_TO_USE or ensure the VPC has discoverable subnets."
+                )
+                raise RuntimeError("No suitable subnets found for automatic selection.")
+            else:
+                print(
+                    f"Automatically selected {len(self.public_subnets)} public and {len(self.private_subnets)} private subnets based on VPC properties."
+                )
+
+            selected_public_subnets = vpc.select_subnets(
+                subnet_type=ec2.SubnetType.PUBLIC, one_per_az=True
+            )
+            private_subnets_egress = vpc.select_subnets(
+                subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS, one_per_az=True
+            )
+
+            if private_subnets_egress.subnets:
+                self.private_subnets.extend(private_subnets_egress.subnets)
+            else:
+                self.node.add_warning(
+                    "No PRIVATE_WITH_EGRESS subnets found in the VPC."
+                )
+
+            try:
+                private_subnets_isolated = vpc.select_subnets(
+                    subnet_type=ec2.SubnetType.PRIVATE_ISOLATED, one_per_az=True
+                )
+            except Exception as e:
+                private_subnets_isolated = []
+                print("Could not find any isolated subnets due to:", e)
+
+            ###
+            combined_subnet_objects = []
+
+            if private_subnets_isolated:
+                if private_subnets_egress.subnets:
+                    # Add the first PRIVATE_WITH_EGRESS subnet
+                    combined_subnet_objects.append(private_subnets_egress.subnets[0])
+            elif not private_subnets_isolated:
+                if private_subnets_egress.subnets:
+                    # Add the first PRIVATE_WITH_EGRESS subnet
+                    combined_subnet_objects.extend(private_subnets_egress.subnets)
+            else:
+                self.node.add_warning(
+                    "No PRIVATE_WITH_EGRESS subnets found to select the first one."
+                )
+
+            # Add all PRIVATE_ISOLATED subnets *except* the first one (if they exist)
+            try:
+                if len(private_subnets_isolated.subnets) > 1:
+                    combined_subnet_objects.extend(private_subnets_isolated.subnets[1:])
+                elif (
+                    private_subnets_isolated.subnets
+                ):  # Only 1 isolated subnet, add a warning if [1:] was desired
+                    self.node.add_warning(
+                        "Only one PRIVATE_ISOLATED subnet found, private_subnets_isolated.subnets[1:] will be empty."
+                    )
+                else:
+                    self.node.add_warning("No PRIVATE_ISOLATED subnets found.")
+            except Exception as e:
+                print("Could not identify private isolated subnets due to:", e)
+
+            # Create an ec2.SelectedSubnets object from the combined private subnet list.
+            selected_private_subnets = vpc.select_subnets(
+                subnets=combined_subnet_objects
+            )
+
+            print("selected_public_subnets:", selected_public_subnets)
+            print("selected_private_subnets:", selected_private_subnets)
+
+            if (
+                len(selected_public_subnets.subnet_ids) < 2
+                or len(selected_private_subnets.subnet_ids) < 2
+            ):
+                raise Exception(
+                    "Need at least two public or private subnets in different availability zones"
+                )
+
+            if not selected_public_subnets and not selected_private_subnets:
+                # If no subnets could be found even with automatic selection, raise an error.
+                # This ensures the stack doesn't proceed if it absolutely needs subnets.
+                print(
+                    "Error: No existing public or private subnets could be found in the VPC for automatic selection. "
+                    "You must either specify subnets in *_SUBNETS_TO_USE or ensure the VPC has discoverable subnets."
+                )
+                raise RuntimeError("No suitable subnets found for automatic selection.")
+            else:
+                self.public_subnets = selected_public_subnets.subnets
+                self.private_subnets = selected_private_subnets.subnets
+                print(
+                    f"Automatically selected {len(self.public_subnets)} public and {len(self.private_subnets)} private subnets based on VPC discovery."
+                )
+
+                print("self.public_subnets:", self.public_subnets)
+                print("self.private_subnets:", self.private_subnets)
+                # Since subnets are now assigned, we can exit this processing block.
+                # The rest of the original code (which iterates *_SUBNETS_TO_USE) will be skipped.
+
+        checked_public_subnets_ctx = get_context_dict("checked_public_subnets")
+        get_context_dict("checked_private_subnets")
+
+        public_subnets_data_for_creation_ctx = get_context_list_of_dicts(
+            "public_subnets_to_create"
+        )
+        private_subnets_data_for_creation_ctx = get_context_list_of_dicts(
+            "private_subnets_to_create"
+        )
+
+        # --- 3. Process Public Subnets ---
+        print("\n--- Processing Public Subnets ---")
+        # Import existing public subnets
+        if checked_public_subnets_ctx:
+            for i, subnet_name in enumerate(PUBLIC_SUBNETS_TO_USE):
+                subnet_info = checked_public_subnets_ctx.get(subnet_name)
+                if subnet_info and subnet_info.get("exists"):
+                    subnet_id = subnet_info.get("id")
+                    if not subnet_id:
+                        raise RuntimeError(
+                            f"Context for existing public subnet '{subnet_name}' is missing 'id'."
+                        )
+                    try:
+                        ec2.Subnet.from_subnet_id(
+                            self,
+                            f"ImportedPublicSubnet{subnet_name.replace('-', '')}{i}",
+                            subnet_id,
+                        )
+                        # self.public_subnets.append(imported_subnet)
+                        print(
+                            f"Imported existing public subnet: {subnet_name} (ID: {subnet_id})"
+                        )
+                    except Exception as e:
+                        raise RuntimeError(
+                            f"Failed to import public subnet '{subnet_name}' with ID '{subnet_id}'. Error: {e}"
+                        )
+
+        # Create new public subnets based on public_subnets_data_for_creation_ctx
+        if public_subnets_data_for_creation_ctx:
+            names_to_create_public = [
+                s["name"] for s in public_subnets_data_for_creation_ctx
+            ]
+            cidrs_to_create_public = [
+                s["cidr"] for s in public_subnets_data_for_creation_ctx
+            ]
+            azs_to_create_public = [
+                s["az"] for s in public_subnets_data_for_creation_ctx
+            ]
+
+            if names_to_create_public:
+                print(
+                    f"Attempting to create {len(names_to_create_public)} new public subnets: {names_to_create_public}"
+                )
+                newly_created_public_subnets, newly_created_public_rts_cfn = (
+                    create_subnets(
+                        self,
+                        vpc,
+                        CDK_PREFIX,
+                        names_to_create_public,
+                        cidrs_to_create_public,
+                        azs_to_create_public,
+                        is_public=True,
+                        internet_gateway_id=EXISTING_IGW_ID,
+                    )
+                )
+                self.public_subnets.extend(newly_created_public_subnets)
+                self.public_route_tables_cfn.extend(newly_created_public_rts_cfn)
+
+        if (
+            not self.public_subnets
+            and not names_to_create_public
+            and not PUBLIC_SUBNETS_TO_USE
+        ):
+            raise Exception("No public subnets found or created, exiting.")
+
+        # --- NAT Gateway Creation/Lookup ---
+        print("Creating NAT gateway/located existing")
+        self.single_nat_gateway_id = None
+
+        nat_gw_id_from_context = SINGLE_NAT_GATEWAY_ID
+
+        if nat_gw_id_from_context:
+            print(
+                f"Using existing NAT Gateway ID from context: {nat_gw_id_from_context}"
+            )
+            self.single_nat_gateway_id = nat_gw_id_from_context
+
+        elif (
+            new_vpc_created
+            and new_vpc_nat_gateways > 0
+            and hasattr(vpc, "nat_gateways")
+            and vpc.nat_gateways
+        ):
+            self.single_nat_gateway_id = vpc.nat_gateways[0].gateway_id
+            print(
+                f"Using NAT Gateway {self.single_nat_gateway_id} created by the new VPC construct."
+            )
+
+        if not self.single_nat_gateway_id:
+            print("Creating a new NAT gateway")
+
+            if hasattr(vpc, "nat_gateways") and vpc.nat_gateways:
+                print("Existing NAT gateway found in vpc")
+                pass
+
+                # If not in context, create a new one, but only if we have a public subnet.
+            elif self.public_subnets:
+                print("NAT Gateway ID not found in context. Creating a new one.")
+                # Place the NAT GW in the first available public subnet
+                first_public_subnet = self.public_subnets[0]
+
+                self.single_nat_gateway_id = create_nat_gateway(
+                    self,
+                    first_public_subnet,
+                    nat_gateway_name=NAT_GATEWAY_NAME,
+                    nat_gateway_id_context_key=SINGLE_NAT_GATEWAY_ID,
+                )
+            else:
+                print(
+                    "WARNING: No public subnets available and NAT gateway not found in existing VPC. Cannot create a NAT Gateway."
+                )
+
+        # --- 4. Process Private Subnets ---
+        print("\n--- Processing Private Subnets ---")
+        # ... (rest of your existing subnet processing logic for checked_private_subnets_ctx) ...
+        # (This part for importing existing subnets remains the same)
+
+        # Create new private subnets
+        if private_subnets_data_for_creation_ctx:
+            names_to_create_private = [
+                s["name"] for s in private_subnets_data_for_creation_ctx
+            ]
+            cidrs_to_create_private = [
+                s["cidr"] for s in private_subnets_data_for_creation_ctx
+            ]
+            azs_to_create_private = [
+                s["az"] for s in private_subnets_data_for_creation_ctx
+            ]
+
+            if names_to_create_private:
+                print(
+                    f"Attempting to create {len(names_to_create_private)} new private subnets: {names_to_create_private}"
+                )
+                # --- CALL THE NEW CREATE_SUBNETS FUNCTION FOR PRIVATE ---
+                # Ensure self.single_nat_gateway_id is available before this call
+                if not self.single_nat_gateway_id:
+                    raise ValueError(
+                        "A single NAT Gateway ID is required for private subnets but was not resolved."
+                    )
+
+                newly_created_private_subnets_cfn, newly_created_private_rts_cfn = (
+                    create_subnets(
+                        self,
+                        vpc,
+                        CDK_PREFIX,
+                        names_to_create_private,
+                        cidrs_to_create_private,
+                        azs_to_create_private,
+                        is_public=False,
+                        single_nat_gateway_id=self.single_nat_gateway_id,  # Pass the single NAT Gateway ID
+                    )
+                )
+                self.private_subnets.extend(newly_created_private_subnets_cfn)
+                self.private_route_tables_cfn.extend(newly_created_private_rts_cfn)
+                print(
+                    f"Successfully defined {len(newly_created_private_subnets_cfn)} new private subnets and their route tables for creation."
+                )
+        else:
+            print(
+                "No private subnets specified for creation in context ('private_subnets_to_create')."
+            )
+
+        # if not self.private_subnets:
+        #     raise Exception("No private subnets found or created, exiting.")
+
+        if (
+            not self.private_subnets
+            and not names_to_create_private
+            and not PRIVATE_SUBNETS_TO_USE
+        ):
+            # This condition might need adjustment for new VPCs.
+            raise Exception("No private subnets found or created, exiting.")
+
+        # --- 5. Sanity Check and Output ---
+        # Output the single NAT Gateway ID for verification
+        if self.single_nat_gateway_id:
+            CfnOutput(
+                self,
+                "SingleNatGatewayId",
+                value=self.single_nat_gateway_id,
+                description="ID of the single NAT Gateway resolved or created.",
+            )
+        elif (
+            NEW_VPC_DEFAULT_NAME
+            and (self.node.try_get_context("new_vpc_nat_gateways") or 1) > 0
+        ):
+            print(
+                "INFO: A new VPC was created with NAT Gateway(s). Their routing is handled by the VPC construct. No single_nat_gateway_id was explicitly set for separate output."
+            )
+        else:
+            out_message = "WARNING: No single NAT Gateway was resolved or created explicitly by the script's logic after VPC setup."
+            print(out_message)
+            raise Exception(out_message)
+
+        # --- Outputs for other stacks/regions ---
+        # These are crucial for cross-stack, cross-region referencing
+
+        self.params = dict()
+        self.params["vpc_id"] = vpc.vpc_id
+        self.params["private_subnets"] = self.private_subnets
+        self.params["private_route_tables"] = self.private_route_tables_cfn
+        self.params["public_subnets"] = self.public_subnets
+        self.params["public_route_tables"] = self.public_route_tables_cfn
+
+        private_subnet_selection = ec2.SubnetSelection(subnets=self.private_subnets)
+        public_subnet_selection = ec2.SubnetSelection(subnets=self.public_subnets)
+
+        for sub in private_subnet_selection.subnets:
+            print(
+                "private subnet:",
+                sub.subnet_id,
+                "is in availability zone:",
+                sub.availability_zone,
+            )
+
+        for sub in public_subnet_selection.subnets:
+            print(
+                "public subnet:",
+                sub.subnet_id,
+                "is in availability zone:",
+                sub.availability_zone,
+            )
+
+        print("Private subnet route tables:", self.private_route_tables_cfn)
+
+        # Add the S3 Gateway Endpoint to the VPC
+        if names_to_create_private:
+            try:
+                s3_gateway_endpoint = vpc.add_gateway_endpoint(
+                    "S3GatewayEndpoint",
+                    service=ec2.GatewayVpcEndpointAwsService.S3,
+                    subnets=[private_subnet_selection],
+                )
+            except Exception as e:
+                print("Could not add S3 gateway endpoint to subnets due to:", e)
+
+            # Output some useful information
+            CfnOutput(
+                self,
+                "VpcIdOutput",
+                value=vpc.vpc_id,
+                description="The ID of the VPC where the S3 Gateway Endpoint is deployed.",
+            )
+            CfnOutput(
+                self,
+                "S3GatewayEndpointService",
+                value=s3_gateway_endpoint.vpc_endpoint_id,
+                description="The id for the S3 Gateway Endpoint.",
+            )  # Specify the S3 service
+
+        # --- IAM Roles ---
+        if USE_CUSTOM_KMS_KEY == "1":
+            kms_key = kms.Key(
+                self,
+                "RedactionSharedKmsKey",
+                alias=CUSTOM_KMS_KEY_NAME,
+                removal_policy=RemovalPolicy.DESTROY,
+            )
+
+            custom_sts_kms_policy_dict = {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Sid": "STSCallerIdentity",
+                        "Effect": "Allow",
+                        "Action": ["sts:GetCallerIdentity"],
+                        "Resource": "*",
+                    },
+                    {
+                        "Sid": "KMSAccess",
+                        "Effect": "Allow",
+                        "Action": ["kms:Encrypt", "kms:Decrypt", "kms:GenerateDataKey"],
+                        "Resource": kms_key.key_arn,  # Use key_arn, as it's the full ARN, safer than key_id
+                    },
+                ],
+            }
+        else:
+            kms_key = None
+
+            custom_sts_kms_policy_dict = {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Sid": "STSCallerIdentity",
+                        "Effect": "Allow",
+                        "Action": ["sts:GetCallerIdentity"],
+                        "Resource": "*",
+                    },
+                    {
+                        "Sid": "KMSSecretsManagerDecrypt",  # Explicitly add decrypt for default key
+                        "Effect": "Allow",
+                        "Action": ["kms:Decrypt"],
+                        "Resource": f"arn:aws:kms:{AWS_REGION}:{AWS_ACCOUNT_ID}:key/aws/secretsmanager",
+                    },
+                ],
+            }
+        custom_sts_kms_policy = json.dumps(custom_sts_kms_policy_dict, indent=4)
+
+        try:
+            codebuild_role_name = CODEBUILD_ROLE_NAME
+
+            if get_context_bool(f"exists:{codebuild_role_name}"):
+                # If exists, lookup/import the role using ARN from context
+                role_arn = get_context_str(f"arn:{codebuild_role_name}")
+                if not role_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{codebuild_role_name}' is required if role exists."
+                    )
+                codebuild_role = iam.Role.from_role_arn(
+                    self, "CodeBuildRole", role_arn=role_arn
+                )
+                print("Using existing CodeBuild role")
+            else:
+                # If not exists, create the role
+                codebuild_role = iam.Role(
+                    self,
+                    "CodeBuildRole",  # Logical ID
+                    role_name=codebuild_role_name,  # Explicit resource name
+                    assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"),
+                )
+                codebuild_role.add_managed_policy(
+                    iam.ManagedPolicy.from_aws_managed_policy_name(
+                        "EC2InstanceProfileForImageBuilderECRContainerBuilds"
+                    )
+                )
+                print("Successfully created new CodeBuild role")
+
+            task_role_name = ECS_TASK_ROLE_NAME
+            if get_context_bool(f"exists:{task_role_name}"):
+                role_arn = get_context_str(f"arn:{task_role_name}")
+                if not role_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{task_role_name}' is required if role exists."
+                    )
+                task_role = iam.Role.from_role_arn(self, "TaskRole", role_arn=role_arn)
+                print("Using existing ECS task role")
+            else:
+                task_role = iam.Role(
+                    self,
+                    "TaskRole",  # Logical ID
+                    role_name=task_role_name,  # Explicit resource name
+                    assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"),
+                )
+                for role in AWS_MANAGED_TASK_ROLES_LIST:
+                    print(f"Adding {role} to policy")
+                    task_role.add_managed_policy(
+                        iam.ManagedPolicy.from_aws_managed_policy_name(f"{role}")
+                    )
+                task_role = add_custom_policies(
+                    self, task_role, custom_policy_text=custom_sts_kms_policy
+                )
+                print("Successfully created new ECS task role")
+
+            execution_role_name = ECS_TASK_EXECUTION_ROLE_NAME
+            if get_context_bool(f"exists:{execution_role_name}"):
+                role_arn = get_context_str(f"arn:{execution_role_name}")
+                if not role_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{execution_role_name}' is required if role exists."
+                    )
+                execution_role = iam.Role.from_role_arn(
+                    self, "ExecutionRole", role_arn=role_arn
+                )
+                print("Using existing ECS execution role")
+            else:
+                execution_role = iam.Role(
+                    self,
+                    "ExecutionRole",  # Logical ID
+                    role_name=execution_role_name,  # Explicit resource name
+                    assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"),
+                )
+                for role in AWS_MANAGED_TASK_ROLES_LIST:
+                    execution_role.add_managed_policy(
+                        iam.ManagedPolicy.from_aws_managed_policy_name(f"{role}")
+                    )
+                execution_role = add_custom_policies(
+                    self, execution_role, custom_policy_text=custom_sts_kms_policy
+                )
+                print("Successfully created new ECS execution role")
+
+        except Exception as e:
+            raise Exception("Failed at IAM role step due to:", e)
+
+        # --- S3 Buckets ---
+        try:
+            log_bucket_name = S3_LOG_CONFIG_BUCKET_NAME
+            if get_context_bool(f"exists:{log_bucket_name}"):
+                bucket = s3.Bucket.from_bucket_name(
+                    self, "LogConfigBucket", bucket_name=log_bucket_name
+                )
+                print("Using existing S3 bucket", log_bucket_name)
+            else:
+                if USE_CUSTOM_KMS_KEY == "1" and isinstance(kms_key, kms.Key):
+                    bucket = s3.Bucket(
+                        self,
+                        "LogConfigBucket",
+                        bucket_name=log_bucket_name,
+                        versioned=False,
+                        removal_policy=RemovalPolicy.DESTROY,
+                        auto_delete_objects=True,
+                        encryption=s3.BucketEncryption.KMS,
+                        encryption_key=kms_key,
+                    )
+                else:
+                    bucket = s3.Bucket(
+                        self,
+                        "LogConfigBucket",
+                        bucket_name=log_bucket_name,
+                        versioned=False,
+                        removal_policy=RemovalPolicy.DESTROY,
+                        auto_delete_objects=True,
+                    )
+
+                print("Created S3 bucket", log_bucket_name)
+
+            # Add policies - this will apply to both created and imported buckets
+            # CDK handles idempotent policy additions
+            bucket.add_to_resource_policy(
+                iam.PolicyStatement(
+                    effect=iam.Effect.ALLOW,
+                    principals=[task_role],  # Pass the role object directly
+                    actions=["s3:GetObject", "s3:PutObject"],
+                    resources=[f"{bucket.bucket_arn}/*"],
+                )
+            )
+            bucket.add_to_resource_policy(
+                iam.PolicyStatement(
+                    effect=iam.Effect.ALLOW,
+                    principals=[task_role],
+                    actions=["s3:ListBucket"],
+                    resources=[bucket.bucket_arn],
+                )
+            )
+
+            output_bucket_name = S3_OUTPUT_BUCKET_NAME
+            if get_context_bool(f"exists:{output_bucket_name}"):
+                output_bucket = s3.Bucket.from_bucket_name(
+                    self, "OutputBucket", bucket_name=output_bucket_name
+                )
+                print("Using existing Output bucket", output_bucket_name)
+            else:
+                if USE_CUSTOM_KMS_KEY == "1" and isinstance(kms_key, kms.Key):
+                    output_bucket = s3.Bucket(
+                        self,
+                        "OutputBucket",
+                        bucket_name=output_bucket_name,
+                        lifecycle_rules=[
+                            s3.LifecycleRule(
+                                expiration=Duration.days(
+                                    int(DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS)
+                                )
+                            )
+                        ],
+                        versioned=False,
+                        removal_policy=RemovalPolicy.DESTROY,
+                        auto_delete_objects=True,
+                        encryption=s3.BucketEncryption.KMS,
+                        encryption_key=kms_key,
+                    )
+                else:
+                    output_bucket = s3.Bucket(
+                        self,
+                        "OutputBucket",
+                        bucket_name=output_bucket_name,
+                        lifecycle_rules=[
+                            s3.LifecycleRule(
+                                expiration=Duration.days(
+                                    int(DAYS_TO_DISPLAY_WHOLE_DOCUMENT_JOBS)
+                                )
+                            )
+                        ],
+                        versioned=False,
+                        removal_policy=RemovalPolicy.DESTROY,
+                        auto_delete_objects=True,
+                    )
+
+                print("Created Output bucket:", output_bucket_name)
+
+            # Add policies to output bucket
+            output_bucket.add_to_resource_policy(
+                iam.PolicyStatement(
+                    effect=iam.Effect.ALLOW,
+                    principals=[task_role],
+                    actions=["s3:GetObject", "s3:PutObject"],
+                    resources=[f"{output_bucket.bucket_arn}/*"],
+                )
+            )
+            output_bucket.add_to_resource_policy(
+                iam.PolicyStatement(
+                    effect=iam.Effect.ALLOW,
+                    principals=[task_role],
+                    actions=["s3:ListBucket"],
+                    resources=[output_bucket.bucket_arn],
+                )
+            )
+
+        except Exception as e:
+            raise Exception("Could not handle S3 buckets due to:", e)
+
+        # --- Elastic Container Registry ---
+        try:
+            full_ecr_repo_name = ECR_CDK_REPO_NAME
+            if get_context_bool(f"exists:{full_ecr_repo_name}"):
+                ecr_repo = ecr.Repository.from_repository_name(
+                    self, "ECRRepo", repository_name=full_ecr_repo_name
+                )
+                print("Using existing ECR repository")
+            else:
+                ecr_repo = ecr.Repository(
+                    self, "ECRRepo", repository_name=full_ecr_repo_name
+                )  # Explicitly set repository_name
+                print("Created ECR repository", full_ecr_repo_name)
+
+            ecr_image_loc = ecr_repo.repository_uri
+        except Exception as e:
+            raise Exception("Could not handle ECR repo due to:", e)
+
+        # --- CODEBUILD ---
+        try:
+            codebuild_project_name = CODEBUILD_PROJECT_NAME
+            if get_context_bool(f"exists:{codebuild_project_name}"):
+                # Lookup CodeBuild project by ARN from context
+                project_arn = get_context_str(f"arn:{codebuild_project_name}")
+                if not project_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{codebuild_project_name}' is required if project exists."
+                    )
+                codebuild_project = codebuild.Project.from_project_arn(
+                    self, "CodeBuildProject", project_arn=project_arn
+                )
+                print("Using existing CodeBuild project")
+            else:
+                codebuild_project = codebuild.Project(
+                    self,
+                    "CodeBuildProject",  # Logical ID
+                    project_name=codebuild_project_name,  # Explicit resource name
+                    source=codebuild.Source.git_hub(
+                        owner=GITHUB_REPO_USERNAME,
+                        repo=GITHUB_REPO_NAME,
+                        branch_or_ref=GITHUB_REPO_BRANCH,
+                    ),
+                    environment=codebuild.BuildEnvironment(
+                        build_image=codebuild.LinuxBuildImage.STANDARD_7_0,
+                        privileged=True,
+                        environment_variables={
+                            "ECR_REPO_NAME": codebuild.BuildEnvironmentVariable(
+                                value=full_ecr_repo_name
+                            ),
+                            "AWS_DEFAULT_REGION": codebuild.BuildEnvironmentVariable(
+                                value=AWS_REGION
+                            ),
+                            "AWS_ACCOUNT_ID": codebuild.BuildEnvironmentVariable(
+                                value=AWS_ACCOUNT_ID
+                            ),
+                            "APP_MODE": codebuild.BuildEnvironmentVariable(
+                                value="gradio"
+                            ),
+                        },
+                    ),
+                    build_spec=codebuild.BuildSpec.from_object(
+                        {
+                            "version": "0.2",
+                            "phases": {
+                                "pre_build": {
+                                    "commands": [
+                                        "echo Logging in to Amazon ECR",
+                                        "aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com",
+                                    ]
+                                },
+                                "build": {
+                                    "commands": [
+                                        "echo Building the Docker image",
+                                        "docker build --build-args APP_MODE=$APP_MODE --target $APP_MODE -t $ECR_REPO_NAME:latest .",
+                                        "docker tag $ECR_REPO_NAME:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$ECR_REPO_NAME:latest",
+                                    ]
+                                },
+                                "post_build": {
+                                    "commands": [
+                                        "echo Pushing the Docker image",
+                                        "docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$ECR_REPO_NAME:latest",
+                                    ]
+                                },
+                            },
+                        }
+                    ),
+                )
+                print("Successfully created CodeBuild project", codebuild_project_name)
+
+            # Grant permissions - applies to both created and imported project role
+            ecr_repo.grant_pull_push(codebuild_project.role)
+
+        except Exception as e:
+            raise Exception("Could not handle Codebuild project due to:", e)
+
+        # --- Security Groups ---
+        try:
+            ecs_security_group_name = ECS_SECURITY_GROUP_NAME
+
+            try:
+                ecs_security_group = ec2.SecurityGroup(
+                    self,
+                    "ECSSecurityGroup",  # Logical ID
+                    security_group_name=ecs_security_group_name,  # Explicit resource name
+                    vpc=vpc,
+                )
+                print(f"Created Security Group: {ecs_security_group_name}")
+            except Exception as e:  # If lookup fails, create
+                print("Failed to create ECS security group due to:", e)
+
+            alb_security_group_name = ALB_NAME_SECURITY_GROUP_NAME
+
+            try:
+                alb_security_group = ec2.SecurityGroup(
+                    self,
+                    "ALBSecurityGroup",  # Logical ID
+                    security_group_name=alb_security_group_name,  # Explicit resource name
+                    vpc=vpc,
+                )
+                print(f"Created Security Group: {alb_security_group_name}")
+            except Exception as e:  # If lookup fails, create
+                print("Failed to create ALB security group due to:", e)
+
+            # Define Ingress Rules - CDK will manage adding/removing these as needed
+            ec2_port_gradio_server_port = ec2.Port.tcp(
+                int(GRADIO_SERVER_PORT)
+            )  # Ensure port is int
+            ecs_security_group.add_ingress_rule(
+                peer=alb_security_group,
+                connection=ec2_port_gradio_server_port,
+                description="ALB traffic",
+            )
+
+            alb_security_group.add_ingress_rule(
+                peer=ec2.Peer.prefix_list("pl-93a247fa"),
+                connection=ec2.Port.all_traffic(),
+                description="CloudFront traffic",
+            )
+
+        except Exception as e:
+            raise Exception("Could not handle security groups due to:", e)
+
+        # --- DynamoDB tables for logs (optional) ---
+
+        if SAVE_LOGS_TO_DYNAMODB == "True":
+            try:
+                print("Creating DynamoDB tables for logs")
+
+                dynamodb.Table(
+                    self,
+                    "RedactionAccessDataTable",
+                    table_name=ACCESS_LOG_DYNAMODB_TABLE_NAME,
+                    partition_key=dynamodb.Attribute(
+                        name="id", type=dynamodb.AttributeType.STRING
+                    ),
+                    billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
+                    removal_policy=RemovalPolicy.DESTROY,
+                )
+
+                dynamodb.Table(
+                    self,
+                    "RedactionFeedbackDataTable",
+                    table_name=FEEDBACK_LOG_DYNAMODB_TABLE_NAME,
+                    partition_key=dynamodb.Attribute(
+                        name="id", type=dynamodb.AttributeType.STRING
+                    ),
+                    billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
+                    removal_policy=RemovalPolicy.DESTROY,
+                )
+
+                dynamodb.Table(
+                    self,
+                    "RedactionUsageDataTable",
+                    table_name=USAGE_LOG_DYNAMODB_TABLE_NAME,
+                    partition_key=dynamodb.Attribute(
+                        name="id", type=dynamodb.AttributeType.STRING
+                    ),
+                    billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
+                    removal_policy=RemovalPolicy.DESTROY,
+                )
+
+            except Exception as e:
+                raise Exception("Could not create DynamoDB tables due to:", e)
+
+        # --- ALB ---
+        try:
+            load_balancer_name = ALB_NAME
+            if len(load_balancer_name) > 32:
+                load_balancer_name = load_balancer_name[-32:]
+            if get_context_bool(f"exists:{load_balancer_name}"):
+                # Lookup ALB by ARN from context
+                alb_arn = get_context_str(f"arn:{load_balancer_name}")
+                if not alb_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{load_balancer_name}' is required if ALB exists."
+                    )
+                alb = elbv2.ApplicationLoadBalancer.from_lookup(
+                    self, "ALB", load_balancer_arn=alb_arn  # Logical ID
+                )
+                print(f"Using existing Application Load Balancer {load_balancer_name}.")
+            else:
+                alb = elbv2.ApplicationLoadBalancer(
+                    self,
+                    "ALB",  # Logical ID
+                    load_balancer_name=load_balancer_name,  # Explicit resource name
+                    vpc=vpc,
+                    internet_facing=True,
+                    security_group=alb_security_group,  # Link to SG
+                    vpc_subnets=public_subnet_selection,  # Link to subnets
+                )
+                print("Successfully created new Application Load Balancer")
+        except Exception as e:
+            raise Exception("Could not handle application load balancer due to:", e)
+
+        # --- Cognito User Pool ---
+        try:
+            if get_context_bool(f"exists:{COGNITO_USER_POOL_NAME}"):
+                # Lookup by ID from context
+                user_pool_id = get_context_str(f"id:{COGNITO_USER_POOL_NAME}")
+                if not user_pool_id:
+                    raise ValueError(
+                        f"Context value 'id:{COGNITO_USER_POOL_NAME}' is required if User Pool exists."
+                    )
+                user_pool = cognito.UserPool.from_user_pool_id(
+                    self, "UserPool", user_pool_id=user_pool_id
+                )
+                print(f"Using existing user pool {user_pool_id}.")
+            else:
+                user_pool = cognito.UserPool(
+                    self,
+                    "UserPool",
+                    user_pool_name=COGNITO_USER_POOL_NAME,
+                    mfa=cognito.Mfa.OFF,  # Adjust as needed
+                    sign_in_aliases=cognito.SignInAliases(email=True),
+                    removal_policy=RemovalPolicy.DESTROY,
+                )  # Adjust as needed
+                print(f"Created new user pool {user_pool.user_pool_id}.")
+
+            # If you're using a certificate, assume that you will be using the ALB Cognito login features. You need different redirect URLs to accept the token that comes from Cognito authentication.
+            if ACM_SSL_CERTIFICATE_ARN:
+                redirect_uris = [
+                    COGNITO_REDIRECTION_URL,
+                    COGNITO_REDIRECTION_URL + "/oauth2/idpresponse",
+                ]
+            else:
+                redirect_uris = [COGNITO_REDIRECTION_URL]
+
+            user_pool_client_name = COGNITO_USER_POOL_CLIENT_NAME
+            if get_context_bool(f"exists:{user_pool_client_name}"):
+                # Lookup by ID from context (requires User Pool object)
+                user_pool_client_id = get_context_str(f"id:{user_pool_client_name}")
+                if not user_pool_client_id:
+                    raise ValueError(
+                        f"Context value 'id:{user_pool_client_name}' is required if User Pool Client exists."
+                    )
+                user_pool_client = cognito.UserPoolClient.from_user_pool_client_id(
+                    self, "UserPoolClient", user_pool_client_id=user_pool_client_id
+                )
+                print(f"Using existing user pool client {user_pool_client_id}.")
+            else:
+                user_pool_client = cognito.UserPoolClient(
+                    self,
+                    "UserPoolClient",
+                    auth_flows=cognito.AuthFlow(
+                        user_srp=True, user_password=True
+                    ),  # Example: enable SRP for secure sign-in
+                    user_pool=user_pool,
+                    generate_secret=True,
+                    user_pool_client_name=user_pool_client_name,
+                    supported_identity_providers=[
+                        cognito.UserPoolClientIdentityProvider.COGNITO
+                    ],
+                    o_auth=cognito.OAuthSettings(
+                        flows=cognito.OAuthFlows(authorization_code_grant=True),
+                        scopes=[
+                            cognito.OAuthScope.OPENID,
+                            cognito.OAuthScope.EMAIL,
+                            cognito.OAuthScope.PROFILE,
+                        ],
+                        callback_urls=redirect_uris,
+                    ),
+                    refresh_token_validity=Duration.minutes(
+                        COGNITO_REFRESH_TOKEN_VALIDITY
+                    ),
+                    id_token_validity=Duration.minutes(COGNITO_ID_TOKEN_VALIDITY),
+                    access_token_validity=Duration.minutes(
+                        COGNITO_ACCESS_TOKEN_VALIDITY
+                    ),
+                )
+
+            CfnOutput(
+                self, "CognitoAppClientId", value=user_pool_client.user_pool_client_id
+            )
+
+            print(
+                f"Created new user pool client {user_pool_client.user_pool_client_id}."
+            )
+
+            # Add a domain to the User Pool (crucial for ALB integration)
+            user_pool_domain = user_pool.add_domain(
+                "UserPoolDomain",
+                cognito_domain=cognito.CognitoDomainOptions(
+                    domain_prefix=COGNITO_USER_POOL_DOMAIN_PREFIX
+                ),
+            )
+
+            # Apply removal_policy to the created UserPoolDomain construct
+            user_pool_domain.apply_removal_policy(policy=RemovalPolicy.DESTROY)
+
+            CfnOutput(
+                self, "CognitoUserPoolLoginUrl", value=user_pool_domain.base_url()
+            )
+
+        except Exception as e:
+            raise Exception("Could not handle Cognito resources due to:", e)
+
+        # --- Secrets Manager Secret ---
+        try:
+            secret_name = COGNITO_USER_POOL_CLIENT_SECRET_NAME
+            if get_context_bool(f"exists:{secret_name}"):
+                # Lookup by name
+                secret = secretsmanager.Secret.from_secret_name_v2(
+                    self, "CognitoSecret", secret_name=secret_name
+                )
+                print("Using existing Secret.")
+            else:
+                if USE_CUSTOM_KMS_KEY == "1" and isinstance(kms_key, kms.Key):
+                    secret = secretsmanager.Secret(
+                        self,
+                        "CognitoSecret",  # Logical ID
+                        secret_name=secret_name,  # Explicit resource name
+                        secret_object_value={
+                            "REDACTION_USER_POOL_ID": SecretValue.unsafe_plain_text(
+                                user_pool.user_pool_id
+                            ),  # Use the CDK attribute
+                            "REDACTION_CLIENT_ID": SecretValue.unsafe_plain_text(
+                                user_pool_client.user_pool_client_id
+                            ),  # Use the CDK attribute
+                            "REDACTION_CLIENT_SECRET": user_pool_client.user_pool_client_secret,  # Use the CDK attribute
+                        },
+                        encryption_key=kms_key,
+                    )
+                else:
+                    secret = secretsmanager.Secret(
+                        self,
+                        "CognitoSecret",  # Logical ID
+                        secret_name=secret_name,  # Explicit resource name
+                        secret_object_value={
+                            "REDACTION_USER_POOL_ID": SecretValue.unsafe_plain_text(
+                                user_pool.user_pool_id
+                            ),  # Use the CDK attribute
+                            "REDACTION_CLIENT_ID": SecretValue.unsafe_plain_text(
+                                user_pool_client.user_pool_client_id
+                            ),  # Use the CDK attribute
+                            "REDACTION_CLIENT_SECRET": user_pool_client.user_pool_client_secret,  # Use the CDK attribute
+                        },
+                    )
+
+                print(
+                    "Created new secret in Secrets Manager for Cognito user pool and related details."
+                )
+
+        except Exception as e:
+            raise Exception("Could not handle Secrets Manager secret due to:", e)
+
+        # --- Fargate Task Definition ---
+        try:
+            fargate_task_definition_name = FARGATE_TASK_DEFINITION_NAME
+
+            read_only_file_system = ECS_READ_ONLY_FILE_SYSTEM == "True"
+
+            if os.path.exists(TASK_DEFINITION_FILE_LOCATION):
+                with open(TASK_DEFINITION_FILE_LOCATION) as f:  # Use correct path
+                    task_def_params = json.load(f)
+                # Need to ensure taskRoleArn and executionRoleArn in JSON are correct ARN strings
+            else:
+                epheremal_storage_volume_name = "appEphemeralVolume"
+
+                task_def_params = {}
+                task_def_params["taskRoleArn"] = (
+                    task_role.role_arn
+                )  # Use CDK role object ARN
+                task_def_params["executionRoleArn"] = (
+                    execution_role.role_arn
+                )  # Use CDK role object ARN
+                task_def_params["memory"] = ECS_TASK_MEMORY_SIZE
+                task_def_params["cpu"] = ECS_TASK_CPU_SIZE
+                container_def = {
+                    "name": full_ecr_repo_name,
+                    "image": ecr_image_loc + ":latest",
+                    "essential": True,
+                    "portMappings": [
+                        {
+                            "containerPort": int(GRADIO_SERVER_PORT),
+                            "hostPort": int(GRADIO_SERVER_PORT),
+                            "protocol": "tcp",
+                            "appProtocol": "http",
+                        }
+                    ],
+                    "logConfiguration": {
+                        "logDriver": "awslogs",
+                        "options": {
+                            "awslogs-group": ECS_LOG_GROUP_NAME,
+                            "awslogs-region": AWS_REGION,
+                            "awslogs-stream-prefix": "ecs",
+                        },
+                    },
+                    "environmentFiles": [
+                        {"value": bucket.bucket_arn + "/config.env", "type": "s3"}
+                    ],
+                    "memoryReservation": int(task_def_params["memory"])
+                    - 512,  # Reserve some memory for the container
+                    "mountPoints": [
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/logs",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/feedback",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/usage",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/input",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/output",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/tmp",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/app/config",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/tmp/matplotlib_cache",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/tmp",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/var/tmp",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/tmp/tld",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/tmp/gradio_tmp",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/.paddlex",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/home/user/.local/share/spacy/data",
+                            "readOnly": False,
+                        },
+                        {
+                            "sourceVolume": epheremal_storage_volume_name,
+                            "containerPath": "/usr/share/tessdata",
+                            "readOnly": False,
+                        },
+                    ],
+                    "readonlyRootFilesystem": read_only_file_system,
+                }
+                task_def_params["containerDefinitions"] = [container_def]
+
+            log_group_name_from_config = task_def_params["containerDefinitions"][0][
+                "logConfiguration"
+            ]["options"]["awslogs-group"]
+
+            cdk_managed_log_group = logs.LogGroup(
+                self,
+                "MyTaskLogGroup",  # CDK Logical ID
+                log_group_name=log_group_name_from_config,
+                retention=logs.RetentionDays.ONE_MONTH,
+                removal_policy=RemovalPolicy.DESTROY,
+            )
+
+            epheremal_storage_volume_cdk_obj = ecs.Volume(
+                name=epheremal_storage_volume_name
+            )
+
+            fargate_task_definition = ecs.FargateTaskDefinition(
+                self,
+                "FargateTaskDefinition",  # Logical ID
+                family=fargate_task_definition_name,
+                cpu=int(task_def_params["cpu"]),
+                memory_limit_mib=int(task_def_params["memory"]),
+                task_role=task_role,
+                execution_role=execution_role,
+                runtime_platform=ecs.RuntimePlatform(
+                    cpu_architecture=ecs.CpuArchitecture.X86_64,
+                    operating_system_family=ecs.OperatingSystemFamily.LINUX,
+                ),
+                ephemeral_storage_gib=21,  # Minimum is 21 GiB
+                volumes=[epheremal_storage_volume_cdk_obj],
+            )
+            print("Fargate task definition defined.")
+
+            # Add container definitions to the task definition object
+            if task_def_params["containerDefinitions"]:
+                container_def_params = task_def_params["containerDefinitions"][0]
+
+                if container_def_params.get("environmentFiles"):
+                    env_files = []
+                    for env_file_param in container_def_params["environmentFiles"]:
+                        # Need to parse the ARN to get the bucket object and key
+                        env_file_arn_parts = env_file_param["value"].split(":::")
+                        bucket_name_and_key = env_file_arn_parts[-1]
+                        env_bucket_name, env_key = bucket_name_and_key.split("/", 1)
+
+                        env_file = ecs.EnvironmentFile.from_bucket(bucket, env_key)
+
+                        env_files.append(env_file)
+
+                container = fargate_task_definition.add_container(
+                    container_def_params["name"],
+                    image=ecs.ContainerImage.from_registry(
+                        container_def_params["image"]
+                    ),
+                    logging=ecs.LogDriver.aws_logs(
+                        stream_prefix=container_def_params["logConfiguration"][
+                            "options"
+                        ]["awslogs-stream-prefix"],
+                        log_group=cdk_managed_log_group,
+                    ),
+                    secrets={
+                        "AWS_USER_POOL_ID": ecs.Secret.from_secrets_manager(
+                            secret, "REDACTION_USER_POOL_ID"
+                        ),
+                        "AWS_CLIENT_ID": ecs.Secret.from_secrets_manager(
+                            secret, "REDACTION_CLIENT_ID"
+                        ),
+                        "AWS_CLIENT_SECRET": ecs.Secret.from_secrets_manager(
+                            secret, "REDACTION_CLIENT_SECRET"
+                        ),
+                    },
+                    environment_files=env_files,
+                    readonly_root_filesystem=read_only_file_system,
+                )
+
+                for port_mapping in container_def_params["portMappings"]:
+                    container.add_port_mappings(
+                        ecs.PortMapping(
+                            container_port=int(port_mapping["containerPort"]),
+                            host_port=int(port_mapping["hostPort"]),
+                            name="port-" + str(port_mapping["containerPort"]),
+                            app_protocol=ecs.AppProtocol.http,
+                            protocol=ecs.Protocol.TCP,
+                        )
+                    )
+
+                container.add_port_mappings(
+                    ecs.PortMapping(
+                        container_port=80,
+                        host_port=80,
+                        name="port-80",
+                        app_protocol=ecs.AppProtocol.http,
+                        protocol=ecs.Protocol.TCP,
+                    )
+                )
+
+                if container_def_params.get("mountPoints"):
+                    mount_points = []
+                    for mount_point in container_def_params["mountPoints"]:
+                        mount_points.append(
+                            ecs.MountPoint(
+                                container_path=mount_point["containerPath"],
+                                read_only=mount_point["readOnly"],
+                                source_volume=epheremal_storage_volume_name,
+                            )
+                        )
+                    container.add_mount_points(*mount_points)
+
+        except Exception as e:
+            raise Exception("Could not handle Fargate task definition due to:", e)
+
+        # --- ECS Cluster ---
+        try:
+            cluster = ecs.Cluster(
+                self,
+                "ECSCluster",  # Logical ID
+                cluster_name=CLUSTER_NAME,  # Explicit resource name
+                enable_fargate_capacity_providers=True,
+                vpc=vpc,
+            )
+            print("Successfully created new ECS cluster")
+        except Exception as e:
+            raise Exception("Could not handle ECS cluster due to:", e)
+
+        # --- ECS Service ---
+        try:
+            ecs_service_name = ECS_SERVICE_NAME
+
+            if ECS_USE_FARGATE_SPOT == "True":
+                use_fargate_spot = "FARGATE_SPOT"
+            if ECS_USE_FARGATE_SPOT == "False":
+                use_fargate_spot = "FARGATE"
+
+            # Check if service exists - from_service_arn or from_service_name (needs cluster)
+            try:
+                # from_service_name is useful if you have the cluster object
+                ecs_service = ecs.FargateService.from_service_attributes(
+                    self,
+                    "ECSService",  # Logical ID
+                    cluster=cluster,  # Requires the cluster object
+                    service_name=ecs_service_name,
+                )
+                print(f"Using existing ECS service {ecs_service_name}.")
+            except Exception:
+                # Service will be created with a count of 0, because you haven't yet actually built the initial Docker container with CodeBuild
+                ecs_service = ecs.FargateService(
+                    self,
+                    "ECSService",  # Logical ID
+                    service_name=ecs_service_name,  # Explicit resource name
+                    platform_version=ecs.FargatePlatformVersion.LATEST,
+                    capacity_provider_strategies=[
+                        ecs.CapacityProviderStrategy(
+                            capacity_provider=use_fargate_spot, base=0, weight=1
+                        )
+                    ],
+                    cluster=cluster,
+                    task_definition=fargate_task_definition,  # Link to TD
+                    security_groups=[ecs_security_group],  # Link to SG
+                    vpc_subnets=ec2.SubnetSelection(
+                        subnets=self.private_subnets
+                    ),  # Link to subnets
+                    min_healthy_percent=0,
+                    max_healthy_percent=100,
+                    desired_count=0,
+                )
+                print("Successfully created new ECS service")
+
+            # Note: Auto-scaling setup would typically go here if needed for the service
+
+        except Exception as e:
+            raise Exception("Could not handle ECS service due to:", e)
+
+        # --- Grant Secret Read Access (Applies to both created and imported roles) ---
+        try:
+            secret.grant_read(task_role)
+            secret.grant_read(execution_role)
+        except Exception as e:
+            raise Exception("Could not grant access to Secrets Manager due to:", e)
+
+        # --- ALB TARGET GROUPS AND LISTENERS ---
+        # This section should primarily define the resources if they are managed by this stack.
+        # CDK handles adding/removing targets and actions on updates.
+        # If they might pre-exist outside the stack, you need lookups.
+        cookie_duration = Duration.hours(12)
+        target_group_name = ALB_TARGET_GROUP_NAME  # Explicit resource name
+        cloudfront_distribution_url = "cloudfront_placeholder.net"  # Need to replace this afterwards with the actual cloudfront_distribution.domain_name
+
+        try:
+            # --- CREATING TARGET GROUPS AND ADDING THE CLOUDFRONT LISTENER RULE ---
+
+            target_group = elbv2.ApplicationTargetGroup(
+                self,
+                "AppTargetGroup",  # Logical ID
+                target_group_name=target_group_name,  # Explicit resource name
+                port=int(GRADIO_SERVER_PORT),  # Ensure port is int
+                protocol=elbv2.ApplicationProtocol.HTTP,
+                targets=[ecs_service],  # Link to ECS Service
+                stickiness_cookie_duration=cookie_duration,
+                vpc=vpc,  # Target Groups need VPC
+            )
+            print(f"ALB target group {target_group_name} defined.")
+
+            # First HTTP
+            listener_port = 80
+            # Check if Listener exists - from_listener_arn or lookup by port/ALB
+
+            http_listener = alb.add_listener(
+                "HttpListener",  # Logical ID
+                port=listener_port,
+                open=False,  # Be cautious with open=True, usually restrict source SG
+            )
+            print(f"ALB listener on port {listener_port} defined.")
+
+            if ACM_SSL_CERTIFICATE_ARN:
+                http_listener.add_action(
+                    "DefaultAction",  # Logical ID for the default action
+                    action=elbv2.ListenerAction.redirect(
+                        protocol="HTTPS",
+                        host="#{host}",
+                        port="443",
+                        path="/#{path}",
+                        query="#{query}",
+                    ),
+                )
+            else:
+                if USE_CLOUDFRONT == "True":
+
+                    # The following default action can be added for the listener after a host header rule is added to the listener manually in the Console as suggested in the above comments.
+                    http_listener.add_action(
+                        "DefaultAction",  # Logical ID for the default action
+                        action=elbv2.ListenerAction.fixed_response(
+                            status_code=403,
+                            content_type="text/plain",
+                            message_body="Access denied",
+                        ),
+                    )
+
+                    # Add the Listener Rule for the specific CloudFront Host Header
+                    http_listener.add_action(
+                        "CloudFrontHostHeaderRule",
+                        action=elbv2.ListenerAction.forward(
+                            target_groups=[target_group],
+                            stickiness_duration=cookie_duration,
+                        ),
+                        priority=1,  # Example priority. Adjust as needed. Lower is evaluated first.
+                        conditions=[
+                            elbv2.ListenerCondition.host_headers(
+                                [cloudfront_distribution_url]
+                            )  # May have to redefine url in console afterwards if not specified in config file
+                        ],
+                    )
+
+                else:
+                    # Add the Listener Rule for the specific CloudFront Host Header
+                    http_listener.add_action(
+                        "CloudFrontHostHeaderRule",
+                        action=elbv2.ListenerAction.forward(
+                            target_groups=[target_group],
+                            stickiness_duration=cookie_duration,
+                        ),
+                    )
+
+                print("Added targets and actions to ALB HTTP listener.")
+
+            # Now the same for HTTPS if you have an ACM certificate
+            if ACM_SSL_CERTIFICATE_ARN:
+                listener_port_https = 443
+                # Check if Listener exists - from_listener_arn or lookup by port/ALB
+
+                https_listener = add_alb_https_listener_with_cert(
+                    self,
+                    "MyHttpsListener",  # Logical ID for the HTTPS listener
+                    alb,
+                    acm_certificate_arn=ACM_SSL_CERTIFICATE_ARN,
+                    default_target_group=target_group,
+                    enable_cognito_auth=True,
+                    cognito_user_pool=user_pool,
+                    cognito_user_pool_client=user_pool_client,
+                    cognito_user_pool_domain=user_pool_domain,
+                    listener_open_to_internet=True,
+                    stickiness_cookie_duration=cookie_duration,
+                )
+
+                if https_listener:
+                    CfnOutput(
+                        self, "HttpsListenerArn", value=https_listener.listener_arn
+                    )
+
+                print(f"ALB listener on port {listener_port_https} defined.")
+
+                # if USE_CLOUDFRONT == 'True':
+                #     # Add default action to the listener
+                #     https_listener.add_action(
+                #         "DefaultAction", # Logical ID for the default action
+                #         action=elbv2.ListenerAction.fixed_response(
+                #             status_code=403,
+                #             content_type="text/plain",
+                #             message_body="Access denied",
+                #         ),
+                #     )
+
+                #     # Add the Listener Rule for the specific CloudFront Host Header
+                #     https_listener.add_action(
+                #         "CloudFrontHostHeaderRuleHTTPS",
+                #         action=elbv2.ListenerAction.forward(target_groups=[target_group],stickiness_duration=cookie_duration),
+                #         priority=1, # Example priority. Adjust as needed. Lower is evaluated first.
+                #         conditions=[
+                #             elbv2.ListenerCondition.host_headers([cloudfront_distribution_url])
+                #         ]
+                #     )
+                # else:
+                #     https_listener.add_action(
+                #         "CloudFrontHostHeaderRuleHTTPS",
+                #         action=elbv2.ListenerAction.forward(target_groups=[target_group],stickiness_duration=cookie_duration))
+
+                print("Added targets and actions to ALB HTTPS listener.")
+
+        except Exception as e:
+            raise Exception(
+                "Could not handle ALB target groups and listeners due to:", e
+            )
+
+        # Create WAF to attach to load balancer
+        try:
+            web_acl_name = LOAD_BALANCER_WEB_ACL_NAME
+            if get_context_bool(f"exists:{web_acl_name}"):
+                # Lookup WAF ACL by ARN from context
+                web_acl_arn = get_context_str(f"arn:{web_acl_name}")
+                if not web_acl_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{web_acl_name}' is required if Web ACL exists."
+                    )
+
+                web_acl = create_web_acl_with_common_rules(
+                    self, web_acl_name, waf_scope="REGIONAL"
+                )  # Assuming it takes scope and name
+                print(f"Handled ALB WAF web ACL {web_acl_name}.")
+            else:
+                web_acl = create_web_acl_with_common_rules(
+                    self, web_acl_name, waf_scope="REGIONAL"
+                )  # Assuming it takes scope and name
+                print(f"Created ALB WAF web ACL {web_acl_name}.")
+
+            wafv2.CfnWebACLAssociation(
+                self,
+                id="alb_waf_association",
+                resource_arn=alb.load_balancer_arn,
+                web_acl_arn=web_acl.attr_arn,
+            )
+
+        except Exception as e:
+            raise Exception("Could not handle create ALB WAF web ACL due to:", e)
+
+        # --- Outputs for other stacks/regions ---
+
+        self.params = dict()
+        self.params["alb_arn_output"] = alb.load_balancer_arn
+        self.params["alb_security_group_id"] = alb_security_group.security_group_id
+        self.params["alb_dns_name"] = alb.load_balancer_dns_name
+
+        CfnOutput(
+            self,
+            "AlbArnOutput",
+            value=alb.load_balancer_arn,
+            description="ARN of the Application Load Balancer",
+            export_name=f"{self.stack_name}-AlbArn",
+        )  # Export name must be unique within the account/region
+
+        CfnOutput(
+            self,
+            "AlbSecurityGroupIdOutput",
+            value=alb_security_group.security_group_id,
+            description="ID of the ALB's Security Group",
+            export_name=f"{self.stack_name}-AlbSgId",
+        )
+        CfnOutput(self, "ALBName", value=alb.load_balancer_name)
+
+        CfnOutput(self, "RegionalAlbDnsName", value=alb.load_balancer_dns_name)
+
+        CfnOutput(self, "CognitoPoolId", value=user_pool.user_pool_id)
+        # Add other outputs if needed
+
+        CfnOutput(self, "ECRRepoUri", value=ecr_repo.repository_uri)
+
+
+# --- CLOUDFRONT DISTRIBUTION in separate stack (us-east-1 required) ---
+class CdkStackCloudfront(Stack):
+
+    def __init__(
+        self,
+        scope: Construct,
+        construct_id: str,
+        alb_arn: str,
+        alb_sec_group_id: str,
+        alb_dns_name: str,
+        **kwargs,
+    ) -> None:
+        super().__init__(scope, construct_id, **kwargs)
+
+        # --- Helper to get context values ---
+        def get_context_bool(key: str, default: bool = False) -> bool:
+            return self.node.try_get_context(key) or default
+
+        def get_context_str(key: str, default: str = None) -> str:
+            return self.node.try_get_context(key) or default
+
+        def get_context_dict(scope: Construct, key: str, default: dict = None) -> dict:
+            return scope.node.try_get_context(key) or default
+
+        print(f"CloudFront Stack: Received ALB ARN: {alb_arn}")
+        print(f"CloudFront Stack: Received ALB Security Group ID: {alb_sec_group_id}")
+
+        if not alb_arn:
+            raise ValueError("ALB ARN must be provided to CloudFront stack")
+        if not alb_sec_group_id:
+            raise ValueError(
+                "ALB Security Group ID must be provided to CloudFront stack"
+            )
+
+        # 2. Import the ALB using its ARN
+        # This imports an existing ALB as a construct in the CloudFront stack's context.
+        # CloudFormation will understand this reference at deploy time.
+        alb = elbv2.ApplicationLoadBalancer.from_application_load_balancer_attributes(
+            self,
+            "ImportedAlb",
+            load_balancer_arn=alb_arn,
+            security_group_id=alb_sec_group_id,
+            load_balancer_dns_name=alb_dns_name,
+        )
+
+        try:
+            web_acl_name = WEB_ACL_NAME
+            if get_context_bool(f"exists:{web_acl_name}"):
+                # Lookup WAF ACL by ARN from context
+                web_acl_arn = get_context_str(f"arn:{web_acl_name}")
+                if not web_acl_arn:
+                    raise ValueError(
+                        f"Context value 'arn:{web_acl_name}' is required if Web ACL exists."
+                    )
+
+                web_acl = create_web_acl_with_common_rules(
+                    self, web_acl_name
+                )  # Assuming it takes scope and name
+                print(f"Handled Cloudfront WAF web ACL {web_acl_name}.")
+            else:
+                web_acl = create_web_acl_with_common_rules(
+                    self, web_acl_name
+                )  # Assuming it takes scope and name
+                print(f"Created Cloudfront WAF web ACL {web_acl_name}.")
+
+            # Add ALB as CloudFront Origin
+            origin = origins.LoadBalancerV2Origin(
+                alb,  # Use the created or looked-up ALB object
+                custom_headers={CUSTOM_HEADER: CUSTOM_HEADER_VALUE},
+                origin_shield_enabled=False,
+                protocol_policy=cloudfront.OriginProtocolPolicy.HTTP_ONLY,
+            )
+
+            if CLOUDFRONT_GEO_RESTRICTION:
+                geo_restrict = cloudfront.GeoRestriction.allowlist(
+                    CLOUDFRONT_GEO_RESTRICTION
+                )
+            else:
+                geo_restrict = None
+
+            cloudfront_distribution = cloudfront.Distribution(
+                self,
+                "CloudFrontDistribution",  # Logical ID
+                comment=CLOUDFRONT_DISTRIBUTION_NAME,  # Use name as comment for easier identification
+                geo_restriction=geo_restrict,
+                default_behavior=cloudfront.BehaviorOptions(
+                    origin=origin,
+                    viewer_protocol_policy=cloudfront.ViewerProtocolPolicy.REDIRECT_TO_HTTPS,
+                    allowed_methods=cloudfront.AllowedMethods.ALLOW_ALL,
+                    cache_policy=cloudfront.CachePolicy.CACHING_DISABLED,
+                    origin_request_policy=cloudfront.OriginRequestPolicy.ALL_VIEWER,
+                ),
+                web_acl_id=web_acl.attr_arn,
+            )
+            print(f"Cloudfront distribution {CLOUDFRONT_DISTRIBUTION_NAME} defined.")
+
+        except Exception as e:
+            raise Exception("Could not handle Cloudfront distribution due to:", e)
+
+        # --- Outputs ---
+        CfnOutput(
+            self, "CloudFrontDistributionURL", value=cloudfront_distribution.domain_name
+        )

cdk/check_resources.py

@@ -0,0 +1,375 @@
+import json
+import os
+from typing import Any, Dict, List
+
+from cdk_config import (  # Import necessary config
+    ALB_NAME,
+    AWS_REGION,
+    CDK_CONFIG_PATH,
+    CDK_FOLDER,
+    CODEBUILD_PROJECT_NAME,
+    CODEBUILD_ROLE_NAME,
+    COGNITO_USER_POOL_CLIENT_NAME,
+    COGNITO_USER_POOL_CLIENT_SECRET_NAME,
+    COGNITO_USER_POOL_NAME,
+    CONTEXT_FILE,
+    ECR_CDK_REPO_NAME,
+    ECS_TASK_EXECUTION_ROLE_NAME,
+    ECS_TASK_ROLE_NAME,
+    PRIVATE_SUBNET_AVAILABILITY_ZONES,
+    PRIVATE_SUBNET_CIDR_BLOCKS,
+    PRIVATE_SUBNETS_TO_USE,
+    PUBLIC_SUBNET_AVAILABILITY_ZONES,
+    PUBLIC_SUBNET_CIDR_BLOCKS,
+    PUBLIC_SUBNETS_TO_USE,
+    S3_LOG_CONFIG_BUCKET_NAME,
+    S3_OUTPUT_BUCKET_NAME,
+    VPC_NAME,
+    WEB_ACL_NAME,
+)
+from cdk_functions import (  # Import your check functions (assuming they use Boto3)
+    _get_existing_subnets_in_vpc,
+    check_alb_exists,
+    check_codebuild_project_exists,
+    check_ecr_repo_exists,
+    check_for_existing_role,
+    check_for_existing_user_pool,
+    check_for_existing_user_pool_client,
+    check_for_secret,
+    check_s3_bucket_exists,
+    check_subnet_exists_by_name,
+    check_web_acl_exists,
+    get_vpc_id_by_name,
+    validate_subnet_creation_parameters,
+    # Add other check functions as needed
+)
+
+cdk_folder = CDK_FOLDER  # <FULL_PATH_TO_CDK_FOLDER_HERE>
+
+# Full path needed to find config file
+os.environ["CDK_CONFIG_PATH"] = cdk_folder + CDK_CONFIG_PATH
+
+
+# --- Helper to parse environment variables into lists ---
+def _get_env_list(env_var_name: str) -> List[str]:
+    """Parses a comma-separated environment variable into a list of strings."""
+    value = env_var_name[1:-1].strip().replace('"', "").replace("'", "")
+    if not value:
+        return []
+    # Split by comma and filter out any empty strings that might result from extra commas
+    return [s.strip() for s in value.split(",") if s.strip()]
+
+
+if PUBLIC_SUBNETS_TO_USE and not isinstance(PUBLIC_SUBNETS_TO_USE, list):
+    PUBLIC_SUBNETS_TO_USE = _get_env_list(PUBLIC_SUBNETS_TO_USE)
+if PRIVATE_SUBNETS_TO_USE and not isinstance(PRIVATE_SUBNETS_TO_USE, list):
+    PRIVATE_SUBNETS_TO_USE = _get_env_list(PRIVATE_SUBNETS_TO_USE)
+if PUBLIC_SUBNET_CIDR_BLOCKS and not isinstance(PUBLIC_SUBNET_CIDR_BLOCKS, list):
+    PUBLIC_SUBNET_CIDR_BLOCKS = _get_env_list(PUBLIC_SUBNET_CIDR_BLOCKS)
+if PUBLIC_SUBNET_AVAILABILITY_ZONES and not isinstance(
+    PUBLIC_SUBNET_AVAILABILITY_ZONES, list
+):
+    PUBLIC_SUBNET_AVAILABILITY_ZONES = _get_env_list(PUBLIC_SUBNET_AVAILABILITY_ZONES)
+if PRIVATE_SUBNET_CIDR_BLOCKS and not isinstance(PRIVATE_SUBNET_CIDR_BLOCKS, list):
+    PRIVATE_SUBNET_CIDR_BLOCKS = _get_env_list(PRIVATE_SUBNET_CIDR_BLOCKS)
+if PRIVATE_SUBNET_AVAILABILITY_ZONES and not isinstance(
+    PRIVATE_SUBNET_AVAILABILITY_ZONES, list
+):
+    PRIVATE_SUBNET_AVAILABILITY_ZONES = _get_env_list(PRIVATE_SUBNET_AVAILABILITY_ZONES)
+
+# Check for the existence of elements in your AWS environment to see if it's necessary to create new versions of the same
+
+
+def check_and_set_context():
+    context_data = {}
+
+    # --- Find the VPC ID first ---
+    if VPC_NAME:
+        print("VPC_NAME:", VPC_NAME)
+        vpc_id, nat_gateways = get_vpc_id_by_name(VPC_NAME)
+
+        # If you expect only one, or one per AZ and you're creating one per AZ in CDK:
+        if nat_gateways:
+            # For simplicity, let's just check if *any* NAT exists in the VPC
+            # A more robust check would match by subnet, AZ, or a specific tag.
+            context_data["exists:NatGateway"] = True
+            context_data["id:NatGateway"] = nat_gateways[0][
+                "NatGatewayId"
+            ]  # Store the ID of the first one found
+        else:
+            context_data["exists:NatGateway"] = False
+            context_data["id:NatGateway"] = None
+
+        if not vpc_id:
+            # If the VPC doesn't exist, you might not be able to check/create subnets.
+            # Decide how to handle this: raise an error, set a flag, etc.
+            raise RuntimeError(
+                f"Required VPC '{VPC_NAME}' not found. Cannot proceed with subnet checks."
+            )
+
+        context_data["vpc_id"] = vpc_id  # Store VPC ID in context
+
+        # SUBNET CHECKS
+        context_data: Dict[str, Any] = {}
+        all_proposed_subnets_data: List[Dict[str, str]] = []
+
+        # Flag to indicate if full validation mode (with CIDR/AZs) is active
+        full_validation_mode = False
+
+        # Determine if full validation mode is possible/desired
+        # It's 'desired' if CIDR/AZs are provided, and their lengths match the name lists.
+        public_ready_for_full_validation = (
+            len(PUBLIC_SUBNETS_TO_USE) > 0
+            and len(PUBLIC_SUBNET_CIDR_BLOCKS) == len(PUBLIC_SUBNETS_TO_USE)
+            and len(PUBLIC_SUBNET_AVAILABILITY_ZONES) == len(PUBLIC_SUBNETS_TO_USE)
+        )
+        private_ready_for_full_validation = (
+            len(PRIVATE_SUBNETS_TO_USE) > 0
+            and len(PRIVATE_SUBNET_CIDR_BLOCKS) == len(PRIVATE_SUBNETS_TO_USE)
+            and len(PRIVATE_SUBNET_AVAILABILITY_ZONES) == len(PRIVATE_SUBNETS_TO_USE)
+        )
+
+        # Activate full validation if *any* type of subnet (public or private) has its full details provided.
+        # You might adjust this logic if you require ALL subnet types to have CIDRs, or NONE.
+        if public_ready_for_full_validation or private_ready_for_full_validation:
+            full_validation_mode = True
+
+            # If some are ready but others aren't, print a warning or raise an error based on your strictness
+            if (
+                public_ready_for_full_validation
+                and not private_ready_for_full_validation
+                and PRIVATE_SUBNETS_TO_USE
+            ):
+                print(
+                    "Warning: Public subnets have CIDRs/AZs, but private subnets do not. Only public will be fully validated/created with CIDRs."
+                )
+            if (
+                private_ready_for_full_validation
+                and not public_ready_for_full_validation
+                and PUBLIC_SUBNETS_TO_USE
+            ):
+                print(
+                    "Warning: Private subnets have CIDRs/AZs, but public subnets do not. Only private will be fully validated/created with CIDRs."
+                )
+
+            # Prepare data for validate_subnet_creation_parameters for all subnets that have full details
+            if public_ready_for_full_validation:
+                for i, name in enumerate(PUBLIC_SUBNETS_TO_USE):
+                    all_proposed_subnets_data.append(
+                        {
+                            "name": name,
+                            "cidr": PUBLIC_SUBNET_CIDR_BLOCKS[i],
+                            "az": PUBLIC_SUBNET_AVAILABILITY_ZONES[i],
+                        }
+                    )
+            if private_ready_for_full_validation:
+                for i, name in enumerate(PRIVATE_SUBNETS_TO_USE):
+                    all_proposed_subnets_data.append(
+                        {
+                            "name": name,
+                            "cidr": PRIVATE_SUBNET_CIDR_BLOCKS[i],
+                            "az": PRIVATE_SUBNET_AVAILABILITY_ZONES[i],
+                        }
+                    )
+
+        print(f"Target VPC ID for Boto3 lookup: {vpc_id}")
+
+        # Fetch all existing subnets in the target VPC once to avoid repeated API calls
+        try:
+            existing_aws_subnets = _get_existing_subnets_in_vpc(vpc_id)
+        except Exception as e:
+            print(f"Failed to fetch existing VPC subnets. Aborting. Error: {e}")
+            raise SystemExit(1)  # Exit immediately if we can't get baseline data
+
+        print("\n--- Running Name-Only Subnet Existence Check Mode ---")
+        # Fallback: check only by name using the existing data
+        checked_public_subnets = {}
+        if PUBLIC_SUBNETS_TO_USE:
+            for subnet_name in PUBLIC_SUBNETS_TO_USE:
+                print("subnet_name:", subnet_name)
+                exists, subnet_id = check_subnet_exists_by_name(
+                    subnet_name, existing_aws_subnets
+                )
+                checked_public_subnets[subnet_name] = {
+                    "exists": exists,
+                    "id": subnet_id,
+                }
+
+                # If the subnet exists, remove it from the proposed subnets list
+                if checked_public_subnets[subnet_name]["exists"] is True:
+                    all_proposed_subnets_data = [
+                        subnet
+                        for subnet in all_proposed_subnets_data
+                        if subnet["name"] != subnet_name
+                    ]
+
+        context_data["checked_public_subnets"] = checked_public_subnets
+
+        checked_private_subnets = {}
+        if PRIVATE_SUBNETS_TO_USE:
+            for subnet_name in PRIVATE_SUBNETS_TO_USE:
+                print("subnet_name:", subnet_name)
+                exists, subnet_id = check_subnet_exists_by_name(
+                    subnet_name, existing_aws_subnets
+                )
+                checked_private_subnets[subnet_name] = {
+                    "exists": exists,
+                    "id": subnet_id,
+                }
+
+                # If the subnet exists, remove it from the proposed subnets list
+                if checked_private_subnets[subnet_name]["exists"] is True:
+                    all_proposed_subnets_data = [
+                        subnet
+                        for subnet in all_proposed_subnets_data
+                        if subnet["name"] != subnet_name
+                    ]
+
+        context_data["checked_private_subnets"] = checked_private_subnets
+
+        print("\nName-only existence subnet check complete.\n")
+
+        if full_validation_mode:
+            print(
+                "\n--- Running in Full Subnet Validation Mode (CIDR/AZs provided) ---"
+            )
+            try:
+                validate_subnet_creation_parameters(
+                    vpc_id, all_proposed_subnets_data, existing_aws_subnets
+                )
+                print("\nPre-synth validation successful. Proceeding with CDK synth.\n")
+
+                # Populate context_data for downstream CDK construct creation
+                context_data["public_subnets_to_create"] = []
+                if public_ready_for_full_validation:
+                    for i, name in enumerate(PUBLIC_SUBNETS_TO_USE):
+                        context_data["public_subnets_to_create"].append(
+                            {
+                                "name": name,
+                                "cidr": PUBLIC_SUBNET_CIDR_BLOCKS[i],
+                                "az": PUBLIC_SUBNET_AVAILABILITY_ZONES[i],
+                                "is_public": True,
+                            }
+                        )
+                context_data["private_subnets_to_create"] = []
+                if private_ready_for_full_validation:
+                    for i, name in enumerate(PRIVATE_SUBNETS_TO_USE):
+                        context_data["private_subnets_to_create"].append(
+                            {
+                                "name": name,
+                                "cidr": PRIVATE_SUBNET_CIDR_BLOCKS[i],
+                                "az": PRIVATE_SUBNET_AVAILABILITY_ZONES[i],
+                                "is_public": False,
+                            }
+                        )
+
+            except (ValueError, Exception) as e:
+                print(f"\nFATAL ERROR: Subnet parameter validation failed: {e}\n")
+                raise SystemExit(1)  # Exit if validation fails
+
+    # Example checks and setting context values
+    # IAM Roles
+    role_name = CODEBUILD_ROLE_NAME
+    exists, _, _ = check_for_existing_role(role_name)
+    context_data[f"exists:{role_name}"] = exists  # Use boolean
+    if exists:
+        _, role_arn, _ = check_for_existing_role(role_name)  # Get ARN if needed
+        context_data[f"arn:{role_name}"] = role_arn
+
+    role_name = ECS_TASK_ROLE_NAME
+    exists, _, _ = check_for_existing_role(role_name)
+    context_data[f"exists:{role_name}"] = exists
+    if exists:
+        _, role_arn, _ = check_for_existing_role(role_name)
+        context_data[f"arn:{role_name}"] = role_arn
+
+    role_name = ECS_TASK_EXECUTION_ROLE_NAME
+    exists, _, _ = check_for_existing_role(role_name)
+    context_data[f"exists:{role_name}"] = exists
+    if exists:
+        _, role_arn, _ = check_for_existing_role(role_name)
+        context_data[f"arn:{role_name}"] = role_arn
+
+    # S3 Buckets
+    bucket_name = S3_LOG_CONFIG_BUCKET_NAME
+    exists, _ = check_s3_bucket_exists(bucket_name)
+    context_data[f"exists:{bucket_name}"] = exists
+    if exists:
+        # You might not need the ARN if using from_bucket_name
+        pass
+
+    output_bucket_name = S3_OUTPUT_BUCKET_NAME
+    exists, _ = check_s3_bucket_exists(output_bucket_name)
+    context_data[f"exists:{output_bucket_name}"] = exists
+    if exists:
+        pass
+
+    # ECR Repository
+    repo_name = ECR_CDK_REPO_NAME
+    exists, _ = check_ecr_repo_exists(repo_name)
+    context_data[f"exists:{repo_name}"] = exists
+    if exists:
+        pass  # from_repository_name is sufficient
+
+    # CodeBuild Project
+    project_name = CODEBUILD_PROJECT_NAME
+    exists, _ = check_codebuild_project_exists(project_name)
+    context_data[f"exists:{project_name}"] = exists
+    if exists:
+        # Need a way to get the ARN from the check function
+        _, project_arn = check_codebuild_project_exists(
+            project_name
+        )  # Assuming it returns ARN
+        context_data[f"arn:{project_name}"] = project_arn
+
+    # ALB (by name lookup)
+    alb_name = ALB_NAME
+    exists, _ = check_alb_exists(alb_name, region_name=AWS_REGION)
+    context_data[f"exists:{alb_name}"] = exists
+    if exists:
+        _, alb_object = check_alb_exists(
+            alb_name, region_name=AWS_REGION
+        )  # Assuming check returns object
+        print("alb_object:", alb_object)
+        context_data[f"arn:{alb_name}"] = alb_object["LoadBalancerArn"]
+
+    # Cognito User Pool (by name)
+    user_pool_name = COGNITO_USER_POOL_NAME
+    exists, user_pool_id, _ = check_for_existing_user_pool(user_pool_name)
+    context_data[f"exists:{user_pool_name}"] = exists
+    if exists:
+        context_data[f"id:{user_pool_name}"] = user_pool_id
+
+    # Cognito User Pool Client (by name and pool ID) - requires User Pool ID from check
+    if user_pool_id:
+        user_pool_id_for_client_check = user_pool_id  # context_data.get(f"id:{user_pool_name}") # Use ID from context
+        user_pool_client_name = COGNITO_USER_POOL_CLIENT_NAME
+        if user_pool_id_for_client_check:
+            exists, client_id, _ = check_for_existing_user_pool_client(
+                user_pool_client_name, user_pool_id_for_client_check
+            )
+            context_data[f"exists:{user_pool_client_name}"] = exists
+            if exists:
+                context_data[f"id:{user_pool_client_name}"] = client_id
+
+    # Secrets Manager Secret (by name)
+    secret_name = COGNITO_USER_POOL_CLIENT_SECRET_NAME
+    exists, _ = check_for_secret(secret_name)
+    context_data[f"exists:{secret_name}"] = exists
+    # You might not need the ARN if using from_secret_name_v2
+
+    # WAF Web ACL (by name and scope)
+    web_acl_name = WEB_ACL_NAME
+    exists, _ = check_web_acl_exists(
+        web_acl_name, scope="CLOUDFRONT"
+    )  # Assuming check returns object
+    context_data[f"exists:{web_acl_name}"] = exists
+    if exists:
+        _, existing_web_acl = check_web_acl_exists(web_acl_name, scope="CLOUDFRONT")
+        context_data[f"arn:{web_acl_name}"] = existing_web_acl.attr_arn
+
+    # Write the context data to the file
+    with open(CONTEXT_FILE, "w") as f:
+        json.dump(context_data, f, indent=2)
+
+    print(f"Context data written to {CONTEXT_FILE}")

cdk/lambda_load_dynamo_logs.py

@@ -0,0 +1,321 @@
+"""
+Lambda handler to export DynamoDB usage log table to CSV and upload to S3.
+
+All inputs are read from environment variables (no argparse).
+Intended to run as an AWS Lambda function; can also be invoked locally
+by setting env vars and calling lambda_handler({}, None).
+
+Environment variables (same semantics as load_dynamo_logs.py CLI):
+  DYNAMODB_TABLE_NAME       - DynamoDB table name (default: redaction_usage)
+  AWS_REGION                - AWS region (optional; if unset, uses AWS_DEFAULT_REGION,
+                              then region from Lambda context ARN, then eu-west-2)
+  OUTPUT_FOLDER             - Local output directory, e.g. /tmp (optional)
+  OUTPUT_FILENAME           - Local output file name (default: dynamodb_logs_export.csv)
+  OUTPUT                    - Full local output path (overrides folder + filename if set).
+                              In Lambda only /tmp is writable; relative paths are auto-resolved to /tmp.
+  FROM_DATE                 - Only include entries on/after this date YYYY-MM-DD (optional)
+  TO_DATE                   - Only include entries on/before this date YYYY-MM-DD (optional)
+  DATE_ATTRIBUTE            - Attribute name for date filtering (default: timestamp)
+  S3_OUTPUT_BUCKET          - S3 bucket for the output CSV (required for upload)
+  S3_OUTPUT_KEY             - S3 object key/path for the output CSV (required for upload)
+"""
+
+import csv
+import datetime
+import os
+from decimal import Decimal
+from io import StringIO
+
+import boto3
+
+
+def _get_region_from_context(context):
+    """Extract region from Lambda context invoked_function_arn (arn:aws:lambda:REGION:ACCOUNT:function:NAME)."""
+    if context is None:
+        return None
+    arn = getattr(context, "invoked_function_arn", None)
+    if not arn or not isinstance(arn, str):
+        return None
+    parts = arn.split(":")
+    if len(parts) >= 4:
+        return parts[3]  # region is 4th segment
+    return None
+
+
+def get_config_from_env(context=None):
+    """Read all settings from environment variables (same inputs as load_dynamo_logs.py).
+    When running in Lambda, context can be passed to derive region from the function ARN if env is not set.
+    """
+    today = datetime.datetime.now().date()
+    one_year_ago = today - datetime.timedelta(days=365)
+
+    table_name = os.environ.get("DYNAMODB_TABLE_NAME") or os.environ.get(
+        "USAGE_LOG_DYNAMODB_TABLE_NAME", "redaction_usage"
+    )
+    region = (
+        os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or ""
+    ).strip()
+    output = os.environ.get("OUTPUT")
+    output_folder = os.environ.get("OUTPUT_FOLDER", "output/")
+    output_filename = os.environ.get("OUTPUT_FILENAME", "dynamodb_logs_export.csv")
+    from_date_str = os.environ.get("FROM_DATE")
+    to_date_str = os.environ.get("TO_DATE")
+    date_attribute = os.environ.get("DATE_ATTRIBUTE", "timestamp")
+    s3_output_bucket = os.environ.get("S3_OUTPUT_BUCKET")
+    s3_output_key = os.environ.get("S3_OUTPUT_KEY")
+
+    if output:
+        local_output_path = output
+    else:
+        folder = output_folder.rstrip("/").rstrip("\\")
+        local_output_path = os.path.join(folder, output_filename)
+
+    # In AWS Lambda only /tmp is writable; resolve relative paths to /tmp to avoid read-only FS errors
+    if os.environ.get("AWS_LAMBDA_FUNCTION_NAME"):
+        resolved = os.path.abspath(local_output_path)
+        if not resolved.startswith("/tmp"):
+            local_output_path = os.path.join(
+                "/tmp", os.path.basename(local_output_path)
+            )
+
+    # Region: env (AWS_REGION / AWS_DEFAULT_REGION) → Lambda context ARN → hardcoded fallback
+    if not region and context is not None:
+        region = _get_region_from_context(context) or ""
+    if not region:
+        region = "FILL IN DEFAULT REGION HERE"
+
+    from_date = None
+    to_date = None
+    if from_date_str:
+        from_date = datetime.datetime.strptime(from_date_str, "%Y-%m-%d").date()
+    if to_date_str:
+        to_date = datetime.datetime.strptime(to_date_str, "%Y-%m-%d").date()
+    if from_date is None and to_date is None:
+        from_date = one_year_ago
+        to_date = today
+    elif from_date is None:
+        from_date = one_year_ago
+    elif to_date is None:
+        to_date = today
+
+    return {
+        "table_name": table_name,
+        "region": region,
+        "local_output_path": local_output_path,
+        "from_date": from_date,
+        "to_date": to_date,
+        "date_attribute": date_attribute,
+        "s3_output_bucket": s3_output_bucket,
+        "s3_output_key": s3_output_key,
+    }
+
+
+# Helper function to convert Decimal to float or int
+def convert_types(item):
+    new_item = {}
+    for key, value in item.items():
+        if isinstance(value, Decimal):
+            new_item[key] = int(value) if value % 1 == 0 else float(value)
+        elif isinstance(value, str):
+            try:
+                dt_obj = datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
+                new_item[key] = dt_obj.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+            except (ValueError, TypeError):
+                new_item[key] = value
+        else:
+            new_item[key] = value
+    return new_item
+
+
+def _parse_item_date(value):
+    """Parse a DynamoDB attribute value to datetime for comparison. Returns None if unparseable."""
+    if value is None:
+        return None
+    if isinstance(value, Decimal):
+        try:
+            return datetime.datetime.utcfromtimestamp(float(value))
+        except (ValueError, OSError):
+            return None
+    if isinstance(value, (int, float)):
+        try:
+            return datetime.datetime.utcfromtimestamp(float(value))
+        except (ValueError, OSError):
+            return None
+    if isinstance(value, str):
+        for fmt in (
+            "%Y-%m-%d %H:%M:%S.%f",
+            "%Y-%m-%d %H:%M:%S",
+            "%Y-%m-%d",
+            "%Y-%m-%dT%H:%M:%S",
+        ):
+            try:
+                return datetime.datetime.strptime(value, fmt)
+            except (ValueError, TypeError):
+                continue
+        try:
+            return datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except (ValueError, TypeError):
+            pass
+    return None
+
+
+def filter_items_by_date(items, from_date, to_date, date_attribute: str):
+    """Return items whose date attribute falls within [from_date, to_date] (inclusive)."""
+    if from_date is None and to_date is None:
+        return items
+    start = datetime.datetime.combine(from_date, datetime.time.min)
+    end = datetime.datetime.combine(to_date, datetime.time.max)
+    filtered = []
+    for item in items:
+        raw = item.get(date_attribute)
+        dt = _parse_item_date(raw)
+        if dt is None:
+            continue
+        if dt.tzinfo:
+            dt = dt.replace(tzinfo=None)
+        if start <= dt <= end:
+            filtered.append(item)
+    return filtered
+
+
+def scan_table(table):
+    """Paginated scan of DynamoDB table."""
+    items = []
+    response = table.scan()
+    items.extend(response["Items"])
+    while "LastEvaluatedKey" in response:
+        response = table.scan(ExclusiveStartKey=response["LastEvaluatedKey"])
+        items.extend(response["Items"])
+    return items
+
+
+def export_to_csv_buffer(items, fields_to_drop=None):
+    """
+    Write items to a CSV in memory; return (csv_string, fieldnames).
+    Use for uploading to S3 without writing to disk.
+    """
+    if not items:
+        return "", []
+
+    drop_set = set(fields_to_drop or [])
+    all_keys = set()
+    for item in items:
+        all_keys.update(item.keys())
+    fieldnames = sorted(list(all_keys - drop_set))
+
+    buf = StringIO()
+    writer = csv.DictWriter(
+        buf, fieldnames=fieldnames, extrasaction="ignore", restval=""
+    )
+    writer.writeheader()
+    for item in items:
+        writer.writerow(convert_types(item))
+    return buf.getvalue(), fieldnames
+
+
+def export_to_csv_file(items, output_path, fields_to_drop=None):
+    """Write items to a CSV file (for optional /tmp or local path)."""
+    csv_string, _ = export_to_csv_buffer(items, fields_to_drop)
+    if not csv_string:
+        return
+    os.makedirs(os.path.dirname(os.path.abspath(output_path)) or ".", exist_ok=True)
+    with open(output_path, "w", newline="", encoding="utf-8-sig") as f:
+        f.write(csv_string)
+
+
+def run_export(config):
+    """
+    Run the full export: scan DynamoDB, filter by date, write CSV (buffer and/or file), upload to S3.
+    """
+    table_name = config["table_name"]
+    region = config["region"]
+    local_output_path = config["local_output_path"]
+    from_date = config["from_date"]
+    to_date = config["to_date"]
+    date_attribute = config["date_attribute"]
+    s3_output_bucket = config["s3_output_bucket"]
+    s3_output_key = config["s3_output_key"]
+
+    if from_date > to_date:
+        raise ValueError("FROM_DATE must be on or before TO_DATE")
+
+    dynamodb = boto3.resource("dynamodb", region_name=region or None)
+    table = dynamodb.Table(table_name)
+
+    items = scan_table(table)
+    items = filter_items_by_date(items, from_date, to_date, date_attribute)
+
+    csv_string, fieldnames = export_to_csv_buffer(items, fields_to_drop=[])
+    result = {
+        "item_count": len(items),
+        "from_date": str(from_date),
+        "to_date": str(to_date),
+        "columns": fieldnames,
+    }
+
+    if csv_string:
+        # Optional: write to local path (e.g. /tmp in Lambda)
+        try:
+            export_to_csv_file(items, local_output_path, fields_to_drop=[])
+            result["local_path"] = local_output_path
+        except Exception as e:
+            result["local_write_error"] = str(e)
+
+        # Upload to S3 if bucket and key are set
+        if s3_output_bucket and s3_output_key:
+            s3 = boto3.client("s3", region_name=region or None)
+            s3.put_object(
+                Bucket=s3_output_bucket,
+                Key=s3_output_key,
+                Body=csv_string.encode("utf-8-sig"),
+                ContentType="text/csv; charset=utf-8",
+            )
+            result["s3_uri"] = f"s3://{s3_output_bucket}/{s3_output_key}"
+        elif s3_output_bucket or s3_output_key:
+            result["s3_skip_reason"] = (
+                "Both S3_OUTPUT_BUCKET and S3_OUTPUT_KEY must be set"
+            )
+
+    return result
+
+
+def lambda_handler(event, context):
+    """
+    AWS Lambda entrypoint. Config is read from environment variables.
+
+    Event is not required for config; it can be used to override env vars
+    (e.g. pass table_name, from_date, to_date, s3_output_bucket, s3_output_key).
+    """
+    config = get_config_from_env(context=context)
+
+    # Optional: allow event to override env-based config
+    if isinstance(event, dict):
+        if event.get("table_name"):
+            config["table_name"] = event["table_name"]
+        if event.get("region"):
+            config["region"] = event["region"]
+        if event.get("from_date"):
+            config["from_date"] = datetime.datetime.strptime(
+                event["from_date"], "%Y-%m-%d"
+            ).date()
+        if event.get("to_date"):
+            config["to_date"] = datetime.datetime.strptime(
+                event["to_date"], "%Y-%m-%d"
+            ).date()
+        if event.get("date_attribute"):
+            config["date_attribute"] = event["date_attribute"]
+        if event.get("s3_output_bucket"):
+            config["s3_output_bucket"] = event["s3_output_bucket"]
+        if event.get("s3_output_key"):
+            config["s3_output_key"] = event["s3_output_key"]
+
+    result = run_export(config)
+    return {"statusCode": 200, "body": result}
+
+
+if __name__ == "__main__":
+    # Allow running locally with env vars set
+    import json
+
+    result = lambda_handler({}, None)
+    print(json.dumps(result, indent=2))

cdk/post_cdk_build_quickstart.py

@@ -0,0 +1,40 @@
+import time
+
+from cdk_config import (
+    CLUSTER_NAME,
+    CODEBUILD_PROJECT_NAME,
+    ECS_SERVICE_NAME,
+    S3_LOG_CONFIG_BUCKET_NAME,
+)
+from cdk_functions import (
+    create_basic_config_env,
+    start_codebuild_build,
+    start_ecs_task,
+    upload_file_to_s3,
+)
+from tqdm import tqdm
+
+# Create basic config.env file that user can use to run the app later. Input is the folder it is saved into.
+create_basic_config_env("config")
+
+# Start codebuild build
+print("Starting CodeBuild project.")
+start_codebuild_build(PROJECT_NAME=CODEBUILD_PROJECT_NAME)
+
+# Upload config.env file to S3 bucket
+upload_file_to_s3(
+    local_file_paths="config/config.env", s3_key="", s3_bucket=S3_LOG_CONFIG_BUCKET_NAME
+)
+
+total_seconds = 660  # 11 minutes
+update_interval = 1  # Update every second
+
+print("Waiting 11 minutes for the CodeBuild container to build.")
+
+# tqdm iterates over a range, and you perform a small sleep in each iteration
+for i in tqdm(range(total_seconds), desc="Building container"):
+    time.sleep(update_interval)
+
+# Start task on ECS
+print("Starting ECS task")
+start_ecs_task(cluster_name=CLUSTER_NAME, service_name=ECS_SERVICE_NAME)

cdk/requirements.txt

@@ -0,0 +1,5 @@
+aws-cdk-lib<=2.250.0
+boto3<=1.42.91
+pandas<=2.3.3
+nodejs<=0.1.1
+python-dotenv<=1.2.2

doc_redaction/__init__.py

@@ -0,0 +1,27 @@
+"""
+doc_redaction package.
+
+This package layer is intentionally thin for now: it preserves existing
+repo-root entrypoints (e.g. `app.py`, `cli_redact.py`) while providing stable
+import paths for PyPI installs.
+"""
+
+from __future__ import annotations
+
+__all__ = ["__version__", "choose_and_run_redactor", "run_redaction"]
+
+try:
+    from importlib.metadata import PackageNotFoundError, version
+
+    try:
+        __version__ = version("doc_redaction")
+    except PackageNotFoundError:  # pragma: no cover
+        __version__ = "0.0.0"
+except Exception:  # pragma: no cover
+    __version__ = "0.0.0"
+
+# Convenience re-exports (package-qualified import surface)
+from doc_redaction.file_redaction import (
+    choose_and_run_redactor,
+    run_redaction,
+)  # noqa: E402

doc_redaction/api.py

@@ -0,0 +1,41 @@
+"""
+Stable programmatic API surface matching Gradio `api_name` values.
+
+This module provides names that exactly match the Gradio endpoint `api_name`
+strings from `app.py`.
+
+By default these names point to the **CLI-first** Python API (`doc_redaction.cli_api`),
+which is the most stable and runnable interface outside Gradio session state.
+"""
+
+from __future__ import annotations
+
+from doc_redaction.cli_api import (
+    apply_review_redactions,
+    combine_review_csvs,
+    combine_review_pdfs,
+    export_review_page_ocr_visualisation,
+    export_review_redaction_overlay,
+    find_duplicate_pages,
+    find_duplicate_tabular,
+    load_and_prepare_documents_or_data,
+    redact_data,
+    redact_document,
+    summarise_document,
+    word_level_ocr_text_search,
+)
+
+__all__ = [
+    "redact_document",
+    "load_and_prepare_documents_or_data",
+    "apply_review_redactions",
+    "export_review_page_ocr_visualisation",
+    "export_review_redaction_overlay",
+    "word_level_ocr_text_search",
+    "redact_data",
+    "find_duplicate_pages",
+    "find_duplicate_tabular",
+    "summarise_document",
+    "combine_review_csvs",
+    "combine_review_pdfs",
+]

doc_redaction/cli_api.py

@@ -0,0 +1,359 @@
+"""
+CLI-first programmatic API surface.
+
+These functions provide a minimal, runnable Python interface that mirrors the
+Gradio `api_name` routes, but executes the underlying workflows via the CLI
+engine (`cli_redact.main(direct_mode_args=...)`).
+
+Return values are lists of output file paths created in `output_dir`.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from pathlib import Path
+from typing import Any, Iterable
+
+
+def _ensure_list(v: str | list[str] | tuple[str, ...]) -> list[str]:
+    if isinstance(v, (list, tuple)):
+        return [str(x) for x in v]
+    return [str(v)]
+
+
+def _snapshot_files(folder: str) -> set[str]:
+    root = Path(folder)
+    if not root.exists():
+        return set()
+    out: set[str] = set()
+    for dirpath, _, filenames in os.walk(root):
+        for name in filenames:
+            out.add(str(Path(dirpath) / name))
+    return out
+
+
+def _default_output_dir(prefix: str) -> str:
+    return tempfile.mkdtemp(prefix=f"doc_redaction_{prefix}_")
+
+
+def _run_cli(
+    *,
+    gradio_api_name: str,
+    overrides: dict[str, Any],
+    output_dir: str | None,
+) -> list[str]:
+    """
+    Run cli_redact.main with merged defaults and return newly created files.
+    """
+    from cli_redact import get_cli_default_args_dict
+    from cli_redact import main as cli_main
+
+    merged = get_cli_default_args_dict()
+    merged.update(overrides)
+
+    if output_dir is None:
+        output_dir = _default_output_dir(gradio_api_name)
+    merged["output_dir"] = str(output_dir)
+
+    before = _snapshot_files(str(output_dir))
+    cli_main(direct_mode_args=merged)
+    after = _snapshot_files(str(output_dir))
+
+    created = sorted(after - before)
+    return created
+
+
+# ---------------------------------------------------------------------------
+# Implemented via CLI engine (matches agent_routes.py)
+# ---------------------------------------------------------------------------
+
+
+def redact_document(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    ocr_method: str | None = None,
+    pii_detector: str | None = None,
+    instruction: str | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """
+    Parity with Gradio `api_name='redact_document'`.
+    Runs CLI task `redact` (PDF/PNG/JPG) or relevant workflow based on file type.
+    """
+    direct: dict[str, Any] = {
+        "task": "redact",
+        "input_file": _ensure_list(input_files),
+    }
+    if ocr_method is not None:
+        direct["ocr_method"] = ocr_method
+    if pii_detector is not None:
+        direct["pii_detector"] = pii_detector
+    if instruction is not None:
+        direct["custom_llm_instructions"] = instruction
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="redact_document", overrides=direct, output_dir=output_dir
+    )
+
+
+def redact_data(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    instruction: str | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='redact_data'` (same CLI task: `redact`)."""
+    direct: dict[str, Any] = {"task": "redact", "input_file": _ensure_list(input_files)}
+    if instruction is not None:
+        direct["custom_llm_instructions"] = instruction
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="redact_data", overrides=direct, output_dir=output_dir
+    )
+
+
+def find_duplicate_pages(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    similarity_threshold: float | None = None,
+    min_word_count: int | None = None,
+    min_consecutive_pages: int | None = None,
+    greedy_match: bool | None = None,
+    combine_pages: bool | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='find_duplicate_pages'`."""
+    direct: dict[str, Any] = {
+        "task": "deduplicate",
+        "duplicate_type": "pages",
+        "input_file": _ensure_list(input_files),
+    }
+    if similarity_threshold is not None:
+        direct["similarity_threshold"] = similarity_threshold
+    if min_word_count is not None:
+        direct["min_word_count"] = min_word_count
+    if min_consecutive_pages is not None:
+        direct["min_consecutive_pages"] = min_consecutive_pages
+    if greedy_match is not None:
+        direct["greedy_match"] = "True" if greedy_match else "False"
+    if combine_pages is not None:
+        direct["combine_pages"] = "True" if combine_pages else "False"
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="find_duplicate_pages", overrides=direct, output_dir=output_dir
+    )
+
+
+def find_duplicate_tabular(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    text_columns: list[str] | None = None,
+    similarity_threshold: float | None = None,
+    min_word_count: int | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='find_duplicate_tabular'`."""
+    direct: dict[str, Any] = {
+        "task": "deduplicate",
+        "duplicate_type": "tabular",
+        "input_file": _ensure_list(input_files),
+    }
+    if text_columns is not None:
+        direct["text_columns"] = list(text_columns)
+    if similarity_threshold is not None:
+        direct["similarity_threshold"] = similarity_threshold
+    if min_word_count is not None:
+        direct["min_word_count"] = min_word_count
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="find_duplicate_tabular",
+        overrides=direct,
+        output_dir=output_dir,
+    )
+
+
+def summarise_document(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='summarise_document'` (CLI task: `summarise`)."""
+    direct: dict[str, Any] = {
+        "task": "summarise",
+        "input_file": _ensure_list(input_files),
+    }
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="summarise_document", overrides=direct, output_dir=output_dir
+    )
+
+
+def combine_review_pdfs(
+    input_files: str | list[str],
+    *,
+    output_dir: str | None = None,
+    overrides: dict[str, Any] | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='combine_review_pdfs'` (CLI task: `combine_review_pdfs`)."""
+    direct: dict[str, Any] = {
+        "task": "combine_review_pdfs",
+        "input_file": _ensure_list(input_files),
+    }
+    if overrides:
+        direct.update(overrides)
+    return _run_cli(
+        gradio_api_name="combine_review_pdfs", overrides=direct, output_dir=output_dir
+    )
+
+
+# ---------------------------------------------------------------------------
+# Implemented without CLI (as per agent_routes.py)
+# ---------------------------------------------------------------------------
+
+
+def combine_review_csvs(
+    input_files: Iterable[str],
+    *,
+    output_dir: str | None = None,
+) -> list[str]:
+    """Parity with Gradio `api_name='combine_review_csvs'`."""
+    from tools.config import OUTPUT_FOLDER
+    from tools.helper_functions import merge_csv_files
+
+    out_dir = str(output_dir or OUTPUT_FOLDER)
+    Path(out_dir).mkdir(parents=True, exist_ok=True)
+    sep = "/" if not out_dir.endswith(("/", "\\")) else ""
+
+    return merge_csv_files([str(p) for p in input_files], output_folder=out_dir + sep)
+
+
+def export_review_redaction_overlay(
+    *,
+    page_image_path: str,
+    boxes: list[dict[str, Any]],
+    page_number: int = 1,
+    doc_base_name: str = "review",
+    review_df_records: list[dict[str, Any]] | None = None,
+    label_abbrev_chars: int | None = None,
+) -> list[str]:
+    """Same behaviour as Gradio ``api_name='page_redaction_review_image'``; Agent API route ``export_review_redaction_overlay``."""
+    import pandas as pd
+
+    from tools.config import OUTPUT_FOLDER
+    from tools.redaction_review import visualise_review_redaction_boxes
+
+    annotator: dict[str, Any] = {"image": page_image_path, "boxes": boxes}
+    review_df = pd.DataFrame(review_df_records) if review_df_records else pd.DataFrame()
+
+    out_dir = str(Path(OUTPUT_FOLDER).expanduser().resolve())
+    Path(out_dir).mkdir(parents=True, exist_ok=True)
+    out_path = visualise_review_redaction_boxes(
+        annotator,
+        review_df=review_df,
+        output_folder=out_dir,
+        page_number=page_number,
+        doc_base_name=doc_base_name,
+        label_abbrev_chars=label_abbrev_chars,
+    )
+    return [out_path] if out_path else []
+
+
+def export_review_page_ocr_visualisation(
+    *,
+    page_image_path: str,
+    ocr_results: dict[str, Any],
+    page_number: int = 1,
+    doc_base_name: str = "review",
+) -> list[str]:
+    """Same behaviour as Gradio ``api_name='page_ocr_review_image'``; Agent API route ``export_review_page_ocr_visualisation``."""
+    from PIL import Image
+
+    from tools.config import OUTPUT_FOLDER
+    from tools.file_redaction import visualise_ocr_words_bounding_boxes
+
+    out_dir = str(Path(OUTPUT_FOLDER).expanduser().resolve())
+    Path(out_dir).mkdir(parents=True, exist_ok=True)
+
+    image_name = f"{str(doc_base_name or 'review')}_page{int(page_number)}.png"
+    log_paths: list[str] = []
+    log_paths = visualise_ocr_words_bounding_boxes(
+        Image.open(page_image_path).convert("RGB"),
+        ocr_results,
+        image_name=image_name,
+        output_folder=out_dir,
+        visualisation_folder="review_ocr_visualisations",
+        add_legend=True,
+        log_files_output_paths=log_paths,
+    )
+    return list(log_paths)
+
+
+# ---------------------------------------------------------------------------
+# Gradio-session-only (no single CLI task)
+# ---------------------------------------------------------------------------
+
+
+def load_and_prepare_documents_or_data(*args: Any, **kwargs: Any) -> list[str]:
+    raise NotImplementedError(
+        "load_and_prepare_documents_or_data is Gradio-session-state driven and is not exposed as a single CLI task."
+    )
+
+
+def apply_review_redactions(
+    pdf_path: str,
+    review_csv_path: str,
+    *,
+    output_dir: str | None = None,
+    input_dir: str | None = None,
+    text_extract_method: str | None = None,
+    efficient_ocr: bool | None = None,
+) -> list[str]:
+    """
+    Headless parity with Gradio ``api_name='apply_review_redactions'``.
+
+    Returns output file paths (redacted PDF, review CSV, logs, etc.).
+    """
+    from tools.simplified_api import run_apply_review_redactions
+
+    r = run_apply_review_redactions(
+        pdf_path=pdf_path,
+        review_csv_path=review_csv_path,
+        output_dir=output_dir,
+        input_dir=input_dir,
+        text_extract_method=text_extract_method,
+        efficient_ocr=efficient_ocr,
+    )
+    return list(r.get("output_paths") or [])
+
+
+def word_level_ocr_text_search(*args: Any, **kwargs: Any) -> list[str]:
+    raise NotImplementedError(
+        "word_level_ocr_text_search is Gradio-session-state driven; no CLI-first equivalent is currently provided."
+    )
+
+
+__all__ = [
+    "redact_document",
+    "load_and_prepare_documents_or_data",
+    "apply_review_redactions",
+    "export_review_page_ocr_visualisation",
+    "export_review_redaction_overlay",
+    "word_level_ocr_text_search",
+    "redact_data",
+    "find_duplicate_pages",
+    "find_duplicate_tabular",
+    "summarise_document",
+    "combine_review_csvs",
+    "combine_review_pdfs",
+]

doc_redaction/cli_redact.py

@@ -0,0 +1,26 @@
+"""
+CLI entrypoint for packaging.
+
+Re-exports the existing repo-root `cli_redact.py` implementation so that
+`pyproject.toml` console scripts can target a stable package path.
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any, Dict
+
+_root_cli = importlib.import_module("cli_redact")
+
+build_cli_argument_parser = getattr(_root_cli, "build_cli_argument_parser")
+get_cli_default_args_dict = getattr(_root_cli, "get_cli_default_args_dict")
+
+
+def main(direct_mode_args: Dict[str, Any] | None = None):
+    # Mirror the root signature but avoid a mutable default.
+    if direct_mode_args is None:
+        direct_mode_args = {}
+    return _root_cli.main(direct_mode_args=direct_mode_args)
+
+
+__all__ = ["build_cli_argument_parser", "get_cli_default_args_dict", "main"]

doc_redaction/data_anonymise.py

@@ -0,0 +1,9 @@
+"""
+Public API wrappers for tabular anonymisation functions.
+"""
+
+from __future__ import annotations
+
+from tools.data_anonymise import anonymise_files_with_open_text
+
+__all__ = ["anonymise_files_with_open_text"]

doc_redaction/example_data/Bold minimalist professional cover letter.docx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c8551ac157f350b2093e5d8c89f68474f613350074201cff6d52d5ed5ec28ff
+size 23992

doc_redaction/example_data/Example-cv-university-graduaty-hr-role-with-photo-2.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:caf00ca5cb06b8019804d1a7eaeceec772607969e8cad6c34d1d583876345b90
+size 116763

doc_redaction/example_data/Partnership-Agreement-Toolkit_0_0.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0db46a784d7aaafb8d02acf8686523dd376400117d07926a5dcb51ceb69e3236
+size 426602

doc_redaction/example_data/Partnership-Agreement-Toolkit_test_deny_list_para_single_spell.csv

@@ -0,0 +1,2 @@
+another  country or territory sign a formel agreement on behalf? of their communities endorsing a
+soster citues international

doc_redaction/example_data/combined_case_notes.csv

@@ -0,0 +1,19 @@
+Date,Social Worker,Client,Case Note
+"January 3, 2023",Jane Smith,Alex D.,"Met with Alex at school following reports of increased absences and declining grades. Alex appeared sullen and avoided eye contact. When prompted about school, Alex expressed feelings of isolation and stated, ""No one gets me."" Scheduled a follow-up meeting to further explore these feelings."
+"January 17, 2023",Jane Smith,Alex D.,"Met with Alex at the community center. Alex displayed sudden outbursts of anger when discussing home life, particularly in relation to a new stepfather. Alex mentioned occasional substance use, but did not specify which substances. Recommended a comprehensive assessment."
+"February 5, 2023",Jane Smith,Alex D.,Home visit conducted. Alex's mother reported frequent arguments at home. She expressed concerns about Alex's new group of friends and late-night outings. Noted potential signs of substance abuse. Suggested family counseling.
+"February 21, 2023",Jane Smith,Alex D.,"Met with Alex alone at my office. Alex appeared more agitated than in previous meetings. There were visible signs of self-harm on Alex's arms. When questioned, Alex became defensive. Immediate referral made to a mental health professional."
+"March 10, 2023",Jane Smith,Alex D.,Attended joint session with Alex and a therapist. Alex shared feelings of hopelessness and admitted to occasional thoughts of self-harm. Therapist recommended a comprehensive mental health evaluation and ongoing therapy.
+"March 25, 2023",Jane Smith,Alex D.,"Received a call from Alex's school about a physical altercation with another student. Met with Alex, who displayed high levels of frustration and admitted to the use of alcohol. Discussed the importance of seeking help and finding positive coping mechanisms. Recommended enrollment in an anger management program."
+"April 15, 2023",Jane Smith,Alex D.,Met with Alex and mother to discuss progress. Alex's mother expressed concerns about Alex's increasing aggression at home. Alex acknowledged the issues but blamed others for provoking the behavior. It was decided that a more intensive intervention may be needed.
+"April 30, 2023",Jane Smith,Alex D.,"Met with Alex and a psychiatrist. Psychiatrist diagnosed Alex with Oppositional Defiant Disorder (ODD) and co-morbid substance use disorder. A treatment plan was discussed, including medication, therapy, and family counseling."
+"May 20, 2023",Jane Smith,Alex D.,"Met with Alex to discuss progress. Alex has started attending group therapy and has shown slight improvements in behavior. Still, concerns remain about substance use. Discussed potential for a short-term residential treatment program."
+"January 3, 2023",Jane Smith,Jamie L.,"Met with Jamie at school after receiving reports of consistent tardiness and decreased participation in class. Jamie appeared withdrawn and exhibited signs of sadness. When asked about feelings, Jamie expressed feeling ""empty"" and ""hopeless"" at times. Scheduled a follow-up meeting to further explore these feelings."
+"January 17, 2023",Jane Smith,Jamie L.,"Met with Jamie at the community center. Jamie shared feelings of low self-worth, mentioning that it's hard to find motivation for daily tasks. Discussed potential triggers and learned about recent family financial struggles. Recommended counseling and possible group therapy for peer support."
+"February 5, 2023",Jane Smith,Jamie L.,Home visit conducted. Jamie's parents shared concerns about Jamie's increasing withdrawal from family activities and lack of interest in hobbies. Parents mentioned that Jamie spends a lot of time alone in the room. Suggested family therapy to open communication channels.
+"February 21, 2023",Jane Smith,Jamie L.,Met with Jamie in my office. Jamie opened up about feelings of isolation and mentioned difficulty sleeping. No signs of self-harm or suicidal ideation were noted. Recommended a comprehensive mental health assessment to better understand the depth of the depression.
+"March 10, 2023",Jane Smith,Jamie L.,"Attended a joint session with Jamie and a therapist. The therapist noted signs of moderate depression. Together, we discussed coping strategies and potential interventions. Jamie showed interest in art therapy."
+"March 25, 2023",Jane Smith,Jamie L.,"Received feedback from Jamie's school that academic performance has slightly improved. However, social interactions remain limited. Encouraged Jamie to join school clubs or groups to foster connection."
+"April 15, 2023",Jane Smith,Jamie L.,"Met with Jamie and parents to discuss progress. Parents have observed slight improvements in mood on some days, but overall, Jamie still appears to struggle. It was decided to explore medication as a potential aid alongside therapy."
+"April 30, 2023",Jane Smith,Jamie L.,Met with Jamie and a psychiatrist. The psychiatrist diagnosed Jamie with Major Depressive Disorder (MDD) and suggested considering antidepressant medication. Discussed the potential benefits and side effects. Jamie and parents will think it over.
+"May 20, 2023",Jane Smith,Jamie L.,"Jamie has started on a low dose of an antidepressant. Initial feedback is positive, with some improvement in mood and energy levels. Will continue monitoring and adjusting as necessary."

doc_redaction/example_data/combined_case_notes.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09300597024591d0b5b4ef97faef12fcceb28fcbb6ea09260bc42f43967753a4
+size 12579

doc_redaction/example_data/doubled_output_joined.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6eeac353164447c2aa429196e1a6ffae4c095d7171e63c2d1cd1966fdf32d1ed
+size 1274719

doc_redaction/example_data/example_of_emails_sent_to_a_professor_before_applying.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed0cd82b5b5826b851ca0e7c102d2d4d27580f7a90de4211a33178a6664d008d
+size 8848

doc_redaction/example_data/example_outputs/Partnership-Agreement-Toolkit_0_0.pdf_ocr_output.csv

@@ -0,0 +1,277 @@
+page,text,left,top,width,height,line
+1,Partnership Agreement,0.516078,0.027879,0.440784,0.032424,1
+1,SisterCities,0.169804,0.033333,0.238431,0.028182,2
+1,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788,3
+1,Toolkit,0.830588,0.07303,0.126667,0.025152,4
+1,Connect globally. Thrive locally.,0.169804,0.08697,0.238824,0.01303,5
+1,Types of Affiliations,0.117255,0.157576,0.241961,0.02,6
+1,Sister City Relationship,0.117647,0.187273,0.196863,0.013939,7
+1,"A Sister City relationship is formed when the mayor or highest elected official (or, if elections",0.117255,0.211212,0.738824,0.013636,8
+1,"do not take place, highest appointed official) from a U.S. community and a community in",0.117647,0.227273,0.70902,0.013939,9
+1,another country or territory sign a formal agreement on behalf of their communities endorsing a,0.117647,0.243636,0.761961,0.013636,10
+1,"""sister city/sister cities"" relationship. Sister city agreements shall be considered active/valid",0.118039,0.259697,0.731373,0.013939,11
+1,unless otherwise indicated by one or both of the respective communities.,0.118039,0.276061,0.58549,0.013636,12
+1,Sister Cities International shall formally recognize only those relationships by cities/members in,0.118039,0.299697,0.758824,0.013636,13
+1,good standing (i.e. who are current on membership dues) in its Membership Directory or on its,0.117647,0.316061,0.754902,0.013636,14
+1,"website. However, Sister Cities International shall not assert as invalid or otherwise impugn the",0.116863,0.332121,0.760784,0.013636,15
+1,legitimacy of those relationships formed by non-members.,0.118039,0.348485,0.466275,0.013636,16
+1,Friendship City,0.118039,0.372121,0.127059,0.013939,17
+1,"A Friendship City or Friendship Cities relationship is often formed by cities as a ""stepping",0.117255,0.395758,0.714118,0.013636,18
+1,"stone"" to a more formal ""Sister City"" agreement. Typically Friendship City agreements are",0.117647,0.411515,0.720392,0.014242,19
+1,referred to as such in the formal documents that are signed. Sister Cities International shall,0.118039,0.428182,0.72549,0.013636,20
+1,recognize Friendship City relationships by members in its Membership Directory and website.,0.118039,0.444242,0.747843,0.013636,21
+1,As per Sister Cities International Board of Directors:,0.117255,0.467879,0.413333,0.013636,22
+1,Sister Cities International will recognize a new sister cities affiliation between a,0.169412,0.492121,0.626667,0.013333,23
+1,"U.S. and an international community, even though another affiliation may exist",0.169412,0.507879,0.625098,0.013636,24
+1,"between that international community and a different U.S. community, only if a",0.169412,0.524545,0.62902,0.013636,25
+1,cooperative agreement among all involved communities is filed with Sister Cities,0.16902,0.540606,0.643137,0.013636,26
+1,"International. If a cooperative agreement is denied, or no response to the request",0.170196,0.556667,0.647843,0.013333,27
+1,"is received within a reasonable amount of time, Sister Cities International will",0.169412,0.57303,0.612157,0.012727,28
+1,recognize the partnership as a friendship city and it will be delineated as such,0.169412,0.589091,0.621176,0.013636,29
+1,with a symbol in the membership directories.,0.168627,0.605455,0.358824,0.013333,30
+1,The cooperative agreement must be sent by the Mayor/County,0.168627,0.628788,0.509412,0.013939,31
+1,"Executive/Governor of the requesting community, and must be sent to the",0.169804,0.645152,0.595294,0.014242,32
+1,Mayor/County Executive/Governor of each of the existing partnership,0.169804,0.661212,0.555294,0.013636,33
+1,communities. Although the Mayor/County Executive/Governor may request input,0.16902,0.677879,0.647451,0.013636,34
+1,"from, or may be given input by, the sister cities program, it is up to the discretion",0.168627,0.693939,0.647059,0.013939,35
+1,of the Mayor/County Executive/Governor to sign the cooperative agreement.,0.16902,0.709697,0.612941,0.013939,36
+1,Although Sister Cities International will help with the cooperative agreement,0.168627,0.726364,0.605882,0.013636,37
+1,"process, it is up to the requesting community to get the agreement signed. Sister",0.169412,0.742121,0.650196,0.013939,38
+1,"Cities International will not, in any way, force a community to ""share"" and sign",0.16902,0.758182,0.623922,0.014242,39
+1,the cooperative agreement.,0.168627,0.774848,0.219216,0.013333,40
+1,"To place a relationship into Emeritus status, the mayor or highest elected official of the U.S.",0.117255,0.798485,0.736471,0.013939,41
+1,community must write a letter to the mayor of the foreign city indicating that they wish to,0.118039,0.814545,0.70902,0.013636,42
+1,"remain sister cities, but understand that the relationship will remain inactive until such time as",0.118039,0.831212,0.747451,0.013333,43
+1,both cities are able to sustain an active relationship. Sister Cities International should be,0.118039,0.847273,0.705098,0.013636,44
+1,informed in writing by the mayor of the U.S. city of the situation. Sister Cities International will,0.118039,0.863333,0.746275,0.013636,45
+2,Partnership Agreement,0.516078,0.027879,0.440784,0.032424,1
+2,SisterCities,0.169804,0.033333,0.238824,0.028182,2
+2,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788,3
+2,Toolkit,0.83098,0.072727,0.127059,0.025455,4
+2,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303,5
+2,then place the partnership into Emeritus Status and will reflect this status in directories and all,0.117255,0.132424,0.751373,0.013333,6
+2,lists of sister city programs.,0.118039,0.148788,0.218431,0.013333,7
+2,"If a community wishes to terminate a sister city relationship, then a letter from the mayor or",0.118431,0.172424,0.732549,0.013333,8
+2,highest elected official of the U.S. city should be sent to the mayor of the sister city. Sister,0.118039,0.188485,0.721569,0.013636,9
+2,Cities International should be informed of this action in writing by the mayor of the U.S. city,0.118039,0.204848,0.72902,0.013333,10
+2,and Sister Cities International will then remove the partnership from its directories and all lists,0.117647,0.221212,0.746275,0.013333,11
+2,of sister city programs. We do not recommend terminating a relationship simply because it is,0.117647,0.237273,0.743529,0.013333,12
+2,"dormant. Many partnerships wax and wane over the years, and in many cases a dormant",0.117647,0.253939,0.713333,0.013333,13
+2,partnership may be reinvigorated by local members years after it has been inactive.,0.118039,0.269697,0.664314,0.013636,14
+2,General Guidelines,0.118039,0.295152,0.231765,0.016061,15
+2,In order for a sister city/county/state partnership to be recognized by Sister Cities International,0.118431,0.324242,0.754902,0.013636,16
+2,"(SCI), the two communities must sign formal documents which clearly endorse the link. This",0.118039,0.340606,0.74,0.013636,17
+2,presumes several key items: that the U.S. community is already a member of SCI and has,0.118039,0.35697,0.718039,0.013636,18
+2,followed proper procedures (e.g. passed a city council resolution declaring the intent to twin,0.117255,0.373333,0.737647,0.013636,19
+2,with the specific city); that both communities share a mutual commitment to the relationship;,0.117255,0.389394,0.740784,0.013636,20
+2,and that both have secured the necessary support structure to build a lasting relationship. You,0.117647,0.405455,0.758039,0.013333,21
+2,should check with your local sister city program to see if they have any additional requirements,0.117647,0.421818,0.760784,0.013636,22
+2,before pursuing a sister city relationship.,0.118039,0.437879,0.323137,0.013636,23
+2,"SCI often refers to these agreements as a ""Sister City Agreement"" or ""Memorandum of",0.118039,0.461515,0.696863,0.013939,24
+2,"Understanding."" However, as the following examples show, the actual name and format of",0.118039,0.477576,0.729804,0.013636,25
+2,your documents is left up to you.,0.117255,0.494242,0.262745,0.013636,26
+2,A few things to keep in mind as you draft your agreement:,0.117255,0.517879,0.463137,0.013636,27
+2,"Your agreement can range from the ceremonial, with language focusing on each city's",0.176471,0.542121,0.69098,0.013939,28
+2,"commitment to fostering understanding, cooperation, and mutual benefit to the precise,",0.176471,0.558485,0.701961,0.013333,29
+2,"with particular areas of interest, specific programs/activities, or more concrete goals",0.176078,0.574848,0.673725,0.013636,30
+2,related to anything from numbers of exchanges to economic development.,0.176863,0.591212,0.596863,0.013636,31
+2,"Don't try to include everything you plan to do. Some specifics, like particular areas of",0.177255,0.620303,0.681176,0.013939,32
+2,"interest or participating institutions are good to include. However, there's no need to",0.176471,0.636667,0.675686,0.013636,33
+2,include all the programs you plan to do if it makes the document too lengthy or limits,0.176863,0.652727,0.678824,0.013939,34
+2,the scope of projects. This is a formal document to establish the relationship; specific,0.176078,0.668788,0.684706,0.013636,35
+2,"tasks, responsibilities, or other nuts-and-bolts text related to implementation or",0.176078,0.685455,0.635686,0.013333,36
+2,administration of the partnership can be expressed more fully in a separate,0.176471,0.701212,0.600392,0.013636,37
+2,memorandum between the respective sister city committees. Your partnership,0.177255,0.717576,0.626667,0.013636,38
+2,agreement is a historical document and should not be dated or limited by being aligned,0.176471,0.733636,0.699216,0.013636,39
+2,with very specific tasks.,0.176078,0.750606,0.190196,0.013333,40
+2,Work with your counterparts. Remember that this is signed by both cities. You should,0.176078,0.779697,0.68549,0.013636,41
+2,share drafts of your agreement with your international partners and solicit feedback on,0.176471,0.795758,0.691765,0.013333,42
+2,what they'd like to see in the agreement. Be flexible to cultural or municipal priorities.,0.176471,0.811818,0.679216,0.013939,43
+2,Ask your counterparts to translate the agreement if it is drafted in English. It is,0.176078,0.841515,0.623137,0.013636,44
+2,important for the citizens of your partner community to be able to read and understand,0.176863,0.857576,0.693725,0.013939,1
+2,the commitment their city has made. Have someone in your own community who,0.176078,0.873939,0.649804,0.013636,2
+3,Partnership Agreement,0.516078,0.027879,0.441176,0.032121,3
+3,SisterCities,0.169804,0.033333,0.239216,0.028182,4
+3,INTERNATIONAL,0.170196,0.06697,0.237255,0.008788,5
+3,Toolkit,0.83098,0.07303,0.126667,0.025152,6
+3,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303,7
+3,speaks that language check the foreign-language version to make sure it mirrors what,0.176471,0.132424,0.688235,0.013333,8
+3,you have in your own agreement.,0.176471,0.148788,0.264706,0.013333,9
+3,Keep it to one page. Ceremonial documents such as these partnership agreements,0.176863,0.178485,0.66549,0.013636,10
+3,work best if they can be posted in their entirety.,0.176078,0.194545,0.380392,0.013636,11
+3,Most sister city agreements include some acknowledgement of the founding principles,0.177255,0.224242,0.694902,0.013636,12
+3,"of the sister city movement- to promote peace through mutual respect, understanding,",0.176471,0.240303,0.698431,0.013333,13
+3,and cooperation.,0.176471,0.25697,0.13451,0.013333,14
+3,Consider using official letterhead and/or other embellishments such as city seals or,0.176863,0.286061,0.665882,0.013333,15
+3,logos to reflect your enhance the document. Sister city agreements are often posted at,0.176863,0.302121,0.695686,0.013636,16
+3,city hall or other municipal offices and should reflect their historical importance,0.176471,0.318485,0.630588,0.013333,17
+3,Look at other agreements your city has signed. These agreements may give you an idea,0.177255,0.347879,0.705098,0.013636,18
+3,"of what is acceptable or possible, and they may be in an easily replicable format. If you",0.176471,0.364242,0.695686,0.013636,19
+3,"cannot access older agreements please contact Sister Cities International, we may",0.176863,0.380303,0.663137,0.013636,20
+3,"have them on file, although we do not have copies of all partnership agreements.",0.176863,0.396667,0.64549,0.013636,21
+3,Documents must be signed by the top elected official of both communities.,0.177255,0.426364,0.601569,0.013333,22
+3,"Check with your mayor, city council, town clerk, et al. to make sure that the agreement",0.176863,0.455758,0.694118,0.013636,23
+3,"is OK with them. The mayor is the one putting his or her name on the paper, and you",0.176863,0.471818,0.677255,0.013333,24
+3,don't want to spend time developing an agreement which will never be signed.,0.176863,0.488182,0.629412,0.013636,25
+3,Official documents are usually signed during a formal ceremony recognizing the,0.176863,0.517576,0.638431,0.013636,26
+3,partnership. Be sure both communities receive a signed set of the official documents,0.177255,0.533939,0.683922,0.013636,27
+3,for their records.,0.176078,0.550606,0.131373,0.010606,28
+3,Remember to send your signed agreement to Sister Cities International. After we,0.177255,0.579697,0.645098,0.013636,29
+3,receive your agreement we will post the relationship in the City Directory and make sure,0.176863,0.595758,0.703137,0.013636,30
+3,it is included in our Annual Membership Directory.,0.176863,0.612121,0.398039,0.013333,31
+3,Remember that each city's sister city program is independent and can impose requirements,0.118431,0.640606,0.736471,0.013939,32
+3,"like the establishment of a committee, a review period, sustainability/funding plan, among",0.118039,0.65697,0.715686,0.013636,33
+3,"others, before sanctioning a sister city agreement. Check with your local program or mayor's",0.117647,0.672727,0.743529,0.014242,34
+3,office to see if this is the case.,0.117647,0.689091,0.241176,0.011515,35
+3,On the following pages you'll find a series of partnership agreements to give you an idea of,0.118039,0.717879,0.728627,0.013939,36
+3,"what is possible. While you should feel free to use some of the formatting and language, we",0.117255,0.734242,0.73451,0.013636,37
+3,encourage you to make your agreement your own and be creative with what you produce. If,0.117647,0.750606,0.737647,0.013636,38
+3,you are unsure about your agreement or want advice you can always solicit feedback by,0.117647,0.766667,0.708627,0.013636,39
+3,sending it to our Membership Director at akaplan@sister-cities.org or contacting us at (202),0.117647,0.782727,0.732157,0.013636,40
+3,347-8630.,0.117647,0.799394,0.080392,0.010303,41
+4,Partnership Agreement,0.516471,0.027879,0.440784,0.032727,1
+4,SisterCities,0.169412,0.033333,0.239608,0.028485,2
+4,INTERNATIONAL,0.170196,0.066667,0.238431,0.009091,3
+4,Toolkit,0.830588,0.072727,0.127843,0.025758,4
+4,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333,5
+4,"jull bubzig 2000 3,312",0.378039,0.291212,0.32549,0.019394,6
+4,ABU DHABI MUNICIPALITY & TOWN PLANNING,0.376471,0.316667,0.327451,0.016667,7
+4,AN AGREEMENT FOR THE ESTABLISHMENT OF,0.260784,0.373636,0.52549,0.012727,8
+4,SISTER CITIES RELATIONSHIP,0.337647,0.393636,0.342745,0.012121,9
+4,BETWEEN,0.454902,0.413636,0.110588,0.011212,10
+4,THE CITY OF ABU DHABI ( U. A.E),0.337255,0.432727,0.375686,0.013939,11
+4,AND,0.487843,0.452727,0.048235,0.011212,12
+4,"HOUSTON, TEXAS ( U.S.A)",0.385882,0.471515,0.298039,0.014848,13
+4,"The Sister City Program, administered by Sister Cities International, was initiated",0.221961,0.525455,0.597255,0.01303,14
+4,By the President of the United States of America in 1956 to encourage greater,0.222745,0.539394,0.561961,0.012727,15
+4,Friendship and understanding between the United States and other nations through,0.222745,0.553333,0.608235,0.012727,16
+4,Direct personal contact: and,0.222745,0.567576,0.20549,0.012424,17
+4,"In order to foster those goals, the people of Abu Dhabi and Houston, in a gesture of",0.222353,0.594242,0.603529,0.012424,18
+4,"Friendship and goodwill, agree to collaborate for the mutual benefit of their",0.222745,0.608182,0.547843,0.01303,19
+4,"Communities by exploring education, economic and cultural opportunities.",0.222353,0.622121,0.541961,0.012121,20
+4,"Abu Dhabi and Houston, sharing a common interest in energy, technology and",0.221569,0.648788,0.574118,0.012424,21
+4,"medicine, and the desire to promote mutual understanding among our citizens do",0.222353,0.66303,0.588235,0.012121,22
+4,"hereby proclaim themselves Sister Cities beginning on the 13th day of March 2001,",0.221961,0.673636,0.594118,0.015758,23
+4,the date of Houston City Council resolution estatblishing the Sister City,0.221961,0.690303,0.519608,0.01303,24
+4,relationship became effective.,0.221569,0.705152,0.217647,0.012424,25
+4,"Signed on this 26 of October 2002, in duplicate in the Arabic and English",0.221569,0.732121,0.533333,0.01303,26
+4,"Languages, both text being equally authentic.",0.221961,0.746667,0.328627,0.012727,27
+4,A,0.344314,0.768485,0.084706,0.030303,28
+4,Sheikh Mohammed bin Butti AI Hamed,0.245882,0.806364,0.366275,0.010909,29
+4,Lee P.Brown,0.729412,0.806364,0.118824,0.010303,30
+4,Mayor of Houston,0.704706,0.823333,0.166667,0.012424,31
+4,Chairman of Abu Dhabi Municipality,0.24549,0.823636,0.342353,0.012727,32
+4,&Town Planning,0.324314,0.841212,0.155686,0.012424,33
+5,Partnership Agreement,0.516078,0.027879,0.441176,0.032424,1
+5,SisterCities,0.169412,0.033333,0.239608,0.028485,2
+5,INTERNATIONAL,0.17098,0.066667,0.237255,0.009091,3
+5,Toolkit,0.83098,0.072727,0.127059,0.025758,4
+5,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333,5
+5,THE CITY OF NEW YORK,0.438824,0.262121,0.240784,0.009697,6
+5,OFFICE OF THE MAYOR,0.450196,0.27697,0.220392,0.009697,7
+5,"NEW YORK, N.Y. 10007",0.461176,0.29303,0.196863,0.010303,8
+5,THE NEW YORK CITY-LONDON SISTER CITY PARTNERSHIP,0.267451,0.355758,0.582745,0.011818,9
+5,Memorandum of Understanding,0.420392,0.371212,0.274902,0.013333,10
+5,The Sister City partnership between New York City and London will foster mutually,0.201176,0.402121,0.674118,0.014242,11
+5,beneficial solutions to common challenges for these two great cosmopolitan entities.,0.201176,0.417273,0.66902,0.013636,12
+5,"Consequently, the Sister City relationship between the two will be one of the most",0.201176,0.432727,0.652549,0.015152,13
+5,"important in their network of global partnerships, as it strives to:",0.201176,0.448182,0.50902,0.015455,14
+5,Encourage and publicize existing exchanges between London and New York City so,0.230588,0.480303,0.671373,0.015152,15
+5,that they can flourish to benefit a wider cross-section of the citizens of both;,0.230588,0.496061,0.602353,0.015152,16
+5,"Support and promote the development of new social, economic, academic and",0.230196,0.512424,0.618431,0.015455,17
+5,community programs to encourage both cities' citizens to share their experiences as a,0.229804,0.527879,0.678039,0.014848,18
+5,medium for learning from one another;,0.229804,0.543636,0.309412,0.013939,19
+5,Generate an improvement of the operation of the cities' various government agencies,0.229804,0.56,0.676078,0.014545,20
+5,by serving as a conduit of information;,0.22902,0.575758,0.307843,0.014848,21
+5,"Identify themes, common to both, that can generate new initiatives to further and",0.229412,0.591818,0.640784,0.015152,22
+5,"nurture the increasingly powerful financial, social and cultural relationships between",0.22902,0.607576,0.671373,0.014242,23
+5,the cities;,0.22902,0.624545,0.076471,0.012424,24
+5,Promote key mayoral priorities relevant to both London and New York City;,0.228627,0.639394,0.608627,0.015152,25
+5,Provide financial or in kind support to community-led programs that advance the,0.228627,0.656061,0.641569,0.013636,26
+5,aims of the Sister City partnership;,0.22902,0.672121,0.275294,0.013636,27
+5,"With the above purposes in mind, the Mayor of the City of New York and the Mayor of",0.198824,0.702424,0.697647,0.014848,28
+5,London solemnly confirm that these two cities are united by an official partnership by the,0.198824,0.718182,0.710196,0.014545,29
+5,protocol of this Memorandum of Understanding.,0.198431,0.733939,0.384314,0.015152,30
+5,This agreement will go into effect from the date of signatures.,0.310196,0.780606,0.488235,0.014545,31
+5,Thedder Rudolph W. Giuliani,0.178824,0.795455,0.244314,0.100909,32
+5,Signed in March of 2001,0.455686,0.796364,0.19451,0.013636,33
+5,Ken Mayor Livingstone,0.672157,0.877576,0.132941,0.029091,34
+5,Mayor,0.311373,0.894848,0.053333,0.012727,35
+5,New York City,0.287843,0.909091,0.121176,0.013333,36
+5,London,0.701961,0.909091,0.061569,0.010606,37
+6,Partnership Agreement,0.515686,0.027576,0.441961,0.03303,1
+6,SisterCities,0.169412,0.03303,0.24,0.028182,2
+6,INTERNATIONAL,0.169804,0.066667,0.238431,0.009091,3
+6,Toolkit,0.83098,0.072727,0.127451,0.025758,4
+6,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333,5
+6,CHIC OF STATE,0.247451,0.190606,0.141961,0.036364,6
+6,City of Long Beach,0.388627,0.196667,0.476471,0.066364,7
+6,California,0.551373,0.257273,0.136471,0.033333,8
+6,Sister City Agreement,0.321961,0.305455,0.378431,0.035152,9
+6,between the,0.464706,0.352727,0.084314,0.009697,10
+6,City of Long Beach,0.38,0.378485,0.252549,0.01697,11
+6,"California, USA",0.4,0.397576,0.21098,0.016061,12
+6,and the,0.48,0.415152,0.053333,0.009091,13
+6,City of San Pablo de Manta,0.321569,0.428788,0.369804,0.01697,14
+6,"Ecuador, South America",0.347451,0.447879,0.317255,0.015152,15
+6,"In accordance with the authorization and approval expressed by the City of Long Beach,",0.261569,0.482121,0.536863,0.012121,16
+6,"California, USA, and the City of San Pablo de Manta, Ecundor, South America, it is declared",0.217647,0.492727,0.581176,0.01303,17
+6,"that a ""Sister City Agreement between the two cities is hereby established for the following",0.217647,0.502727,0.581569,0.012121,18
+6,purposes:,0.216863,0.516061,0.058039,0.009394,19
+6,(1) to promote and expand the effective and mutually beneficial cooperation between,0.278824,0.532727,0.520392,0.012424,20
+6,the people of Long Beach and the people of San Pablo de Manta; and,0.218039,0.543636,0.40549,0.012424,21
+6,"(2) to promote international goodwill, understanding, and expanded business",0.279216,0.56303,0.520784,0.012424,22
+6,"relations between the two cities and their respective nations by the exchange of people, ideas, and",0.218039,0.573636,0.581569,0.012121,23
+6,"information in a unide variety of economic, social, cultural, municipal, environmental,",0.218039,0.584242,0.581176,0.012121,24
+6,"professional, technical, youth, and other endeavors; and",0.217647,0.594848,0.333333,0.012121,25
+6,"(3) to foster and encourage charitable, scientific, trade and commerce, literary and",0.279608,0.613939,0.520784,0.012727,26
+6,educational activities between the two cities;,0.218039,0.625455,0.265882,0.009697,27
+6,This Sister City Agreement shall be officially established and shall become effective when,0.263137,0.644545,0.536863,0.012727,28
+6,"this document has been duly executed by the Mayor of Long Beach, California, USA, and the",0.218824,0.654848,0.581961,0.012424,29
+6,"Mayor of San Pablo de Manta, Ecundor, South America.",0.218431,0.665758,0.338824,0.012121,30
+6,STATE OFFICE,0.276471,0.713636,0.050588,0.048788,31
+6,Beverly 0 Neill,0.587451,0.736667,0.121961,0.013636,32
+6,"Mayor, City of Long Beach",0.542353,0.751212,0.21098,0.013636,33
+6,"California, USA",0.582745,0.765758,0.125098,0.01303,34
+6,10.2aulus,0.490588,0.771818,0.220392,0.062424,35
+6,Ing. Jorge O. Zambrano Cedeño,0.527059,0.825152,0.242745,0.013333,36
+6,"Mayor, City of San Pablo de Manta",0.505098,0.839394,0.277647,0.013636,37
+6,"Ecuador, South America",0.551765,0.854242,0.188235,0.011818,38
+6,"Dated: September 19, 2000",0.544706,0.883333,0.202745,0.01303,39
+7,Partnership Agreement,0.516078,0.027879,0.441176,0.032424,1
+7,SisterCities,0.169412,0.03303,0.24,0.028485,2
+7,INTERNATIONAL,0.170196,0.066667,0.237647,0.009091,3
+7,Toolkit,0.83098,0.072727,0.127451,0.025758,4
+7,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333,5
+7,REAFFIRMATION OF SISTER CITIES DECLARATION,0.324706,0.165152,0.483529,0.013939,6
+7,adopted by,0.2,0.213333,0.080392,0.013636,7
+7,THE HONORABLE RICHARD M. DALEY,0.396078,0.214242,0.335686,0.012424,8
+7,MAYOR OF CHICAGO,0.472549,0.231212,0.18549,0.011515,9
+7,and,0.199608,0.260909,0.026275,0.010606,10
+7,THE HONORABLE ZHANG RONGMAO,0.401961,0.261212,0.323137,0.011212,11
+7,MAYOR OF SHENYANG,0.463529,0.273636,0.202353,0.011212,12
+7,ON,0.551765,0.298182,0.026667,0.011515,13
+7,"JUNE 5, 1995",0.500392,0.323636,0.128235,0.014848,14
+7,"On this the tenth anniversary of the signing of a sister city agreement, in order to further",0.255686,0.36303,0.67098,0.015152,15
+7,the traditional links of friendship between Chicago and Shenyang and to reaffirm their mutual,0.198824,0.378788,0.727843,0.015455,16
+7,"aspiration to work in unison for the benefit of their cities and nations, the Honorable Mayor",0.199608,0.394848,0.727843,0.014848,17
+7,"Richard M. Daley, Mayor of the City of Chicago, and the Honorable Zhang Rongmao, Mayor",0.199216,0.411212,0.727451,0.014242,18
+7,"of the City of Shenyang, on this fifth day of June 1995, do hereby acknowledge and reaffirm the",0.199216,0.42697,0.72549,0.014848,19
+7,sister cities agreement between the City of Chicago and the City of Shenyang.,0.199608,0.443636,0.57451,0.014242,20
+7,"The City of Chicago and the City of Shenyang on the basis of friendly cooperation,",0.256078,0.473939,0.665098,0.015152,21
+7,equality and mutual benefit will continue to develop a sister cities relationship to promote and,0.2,0.490303,0.724706,0.014242,22
+7,broaden economic cooperation and cultural exchanges between the two cities.,0.199216,0.506061,0.57451,0.014242,23
+7,The two cities do hereby declare their interest in exploring the establishment of business,0.255294,0.537273,0.668235,0.015455,24
+7,and trade relations between Chicago and Shenyang.,0.198824,0.554545,0.387843,0.013636,25
+7,"In addition, exchanges will be promoted in the area of the arts such as exhibits, music,",0.254118,0.583939,0.666667,0.015455,26
+7,dance and other cultural activities.,0.198431,0.601212,0.256471,0.010606,27
+7,"In addition, exchanges will be promoted in education and the establishment of contacts",0.254118,0.630303,0.668627,0.015758,28
+7,within educational institutions encouraged.,0.198824,0.647273,0.32,0.014242,29
+7,"In addition, we declare our intention to promote exchanges in such fields as science and",0.253725,0.678182,0.668627,0.014848,30
+7,"technology, sports, health, youth and any areas that will contribute to the prosperity and the",0.198039,0.693636,0.722745,0.015152,31
+7,further development of friendship between the people of our two cities.,0.194902,0.711515,0.525098,0.013636,32
+7,3h.5.,0.593725,0.750606,0.218039,0.06303,33
+7,THE HONORABLE ZHANG RONGMAO,0.588627,0.819394,0.287843,0.011818,34
+7,THE HONORABLE RICHARD M. DALEY,0.197255,0.821515,0.303529,0.010606,35
+7,MAYOR OF SHENYANG,0.587451,0.835455,0.177647,0.010303,36
+7,MAYOR OF CHICAGO,0.195686,0.835758,0.164706,0.010606,37

doc_redaction/example_data/example_outputs/Partnership-Agreement-Toolkit_0_0.pdf_review_file.csv

@@ -0,0 +1,77 @@
+image,page,label,color,xmin,ymin,xmax,ymax,id,text
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_0.png,1,ADDRESS,"(0, 0, 0)",0.598431,0.524545,0.63098,0.535455,EG3nykuwvxbk,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_0.png,1,ADDRESS,"(0, 0, 0)",0.820392,0.798485,0.854118,0.809394,jy1R42e6phNz,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_0.png,1,ADDRESS,"(0, 0, 0)",0.433333,0.863333,0.46549,0.873939,9sbrsroLfZy0,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_1.png,2,ADDRESS,"(0, 0, 0)",0.354118,0.188788,0.386275,0.199697,k7bWBsQQchJZ,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_1.png,2,ADDRESS,"(0, 0, 0)",0.780392,0.204848,0.812941,0.215758,peo6UqIxrjmR,U.S.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_2.png,3,EMAIL,"(0, 0, 0)",0.447843,0.78303,0.648627,0.796667,DIfz0LenOtQv,akaplan@sister-cities.org
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_2.png,3,PHONE,"(0, 0, 0)",0.809804,0.78303,0.850196,0.796667,odJdySe9XrAn,(202)
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_2.png,3,PHONE,"(0, 0, 0)",0.117647,0.799394,0.198431,0.809697,iURSkUM7BbUG,347-8630
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.637647,0.432727,0.712941,0.44697,fRxAD9qm856s,U. A.E
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.489412,0.43303,0.614902,0.444545,qzRFPlNbslpH,ABU DHABI
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.385882,0.472121,0.593725,0.486364,v1uLbGsofN1f,"HOUSTON, TEXAS"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.392549,0.539697,0.573725,0.549394,MvbPQiHvSdL7,United States of America
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.539216,0.553333,0.635686,0.563333,05U3cgj5w9PY,United States
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.534902,0.594242,0.615294,0.603939,uHMikyBlMq5f,Abu Dhabi
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.651373,0.594242,0.717255,0.605455,XNUE0GopIBaf,Houston
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.221569,0.65,0.301176,0.659697,6FjbNu2CGA9n,Abu Dhabi
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.337647,0.65,0.404314,0.660606,Yvmm2225ityu,Houston
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,HANDWRITING,"(0, 0, 0)",0.344314,0.768485,0.42902,0.798788,EwTcqq7PENU8,A
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.245882,0.806364,0.612549,0.817576,Mj4gqwbgsZWp,Sheikh Mohammed bin Butti AI Hamed
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.52,0.806364,0.612549,0.81697,RXYOVgLwq8Ke,AI Hamed
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.729412,0.806364,0.848235,0.816667,REPZhwFWGoTc,Lee P.Brown
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,NAME,"(0, 0, 0)",0.245882,0.806667,0.51451,0.817576,rFdxMRFRWLRJ,Sheikh Mohammed bin Butti
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_3.png,4,ADDRESS,"(0, 0, 0)",0.366667,0.823939,0.465098,0.834242,5iYCxRGdPG1i,Abu Dhabi
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.577647,0.262121,0.68,0.271515,3ZR43H3yYNdy,NEW YORK
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.461176,0.29303,0.555294,0.303333,WNoitmR9A6lu,NEW YORK
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.461176,0.29303,0.658039,0.303333,HjrhxMQhovlF,NEW YORK N.Y. 10007
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.563137,0.29303,0.658039,0.302121,nPN7g7UcnX4u,N.Y. 10007
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.314118,0.356667,0.42549,0.367576,ZoJf29CB3Wrq,NEW YORK
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.655294,0.480909,0.718431,0.491515,iezAqmD2ilnb,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.708627,0.639394,0.837255,0.652727,tWAuJEQVpfhi,New York City
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.60902,0.64,0.67098,0.650606,NaW3mmmlhMW9,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.667059,0.702727,0.751373,0.713636,pgMiwuMiBp8B,New York
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.198824,0.720303,0.261569,0.731212,fPvElSFZFRoL,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,HANDWRITING,"(0, 0, 0)",0.178824,0.795455,0.281961,0.896364,DfniF7P2bXAw,Thedder
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,NAME,"(0, 0, 0)",0.178824,0.795455,0.423529,0.896364,QwnWsAeslO5f,Thedder Rudolph W. Giuliani
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,NAME - ADDRESS,"(0, 0, 0)",0.672157,0.877576,0.80549,0.891212,Vdp95SShYOEO,Ken Livingstone
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.710196,0.877576,0.80549,0.891212,H5DGqsucPAjc,Livingstone
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,NAME,"(0, 0, 0)",0.672157,0.877879,0.705098,0.888182,qotGtnMbhAJr,Ken
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.287843,0.909091,0.40902,0.922727,sFX0tNJJzpE5,New York City
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_4.png,5,ADDRESS,"(0, 0, 0)",0.701961,0.909091,0.763922,0.919697,2xFbVTbxiOhC,London
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.55451,0.203636,0.86549,0.258485,Nfe3WTBembGQ,Long Beach
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.551373,0.257273,0.687843,0.290606,kndQY5X4itc8,California
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.558824,0.397879,0.611373,0.410303,B5vq8yhWLeOg,USA
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.425882,0.429091,0.691373,0.441818,OtNgqUkoEaZb,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.347451,0.447879,0.665098,0.46303,Q52VzBx2SWNF,"Ecuador, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.724314,0.482121,0.798431,0.493939,O7gd9ywvKsKh,"Long Beach,"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.425098,0.49303,0.506275,0.502727,DzYr3xrM8Tvv,San Pablo de
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.425098,0.49303,0.715294,0.50303,iZ0knpQD54UU,"San Pablo de Manta, Ecundor, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.509804,0.49303,0.715294,0.50303,pZnYGzr7Pwsl,"Manta, Ecundor, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.217647,0.493333,0.321961,0.504242,r7Aar8FNQF6D,"California, USA"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.471765,0.543636,0.596863,0.553939,zg9uBDlSuuA1,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.295294,0.544242,0.36549,0.556061,A0OY6RjMEocW,Long Beach
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.563137,0.655152,0.748627,0.667576,HQlTdEUhOCgI,"Long Beach, California, USA"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.463529,0.665758,0.557255,0.674848,bCN9b7kJw0Ik,South America
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.277647,0.666061,0.403529,0.676061,qffN3bDgWRMk,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.587451,0.736667,0.709804,0.750303,eqMENFw5mbnL,Beverly 0 Neill
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.663137,0.751212,0.753333,0.764545,POqPQVBCES8h,Long Beach
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.582745,0.765758,0.708235,0.779091,mjrjsSMOxwaY,"California, USA"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,HANDWRITING,"(0, 0, 0)",0.490588,0.771818,0.71098,0.834242,xL8dSawihWuY,10.2aulus
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,NAME,"(0, 0, 0)",0.559608,0.825152,0.769804,0.838485,fHyvwmbOgLMJ,Jorge O. Zambrano Cedeño
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.624314,0.839394,0.782745,0.850303,zGhskyehufSv,San Pablo de Manta
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_5.png,6,ADDRESS,"(0, 0, 0)",0.551765,0.854242,0.74,0.866061,dSPXmtb8M4nt,"Ecuador, South America"
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.556471,0.215152,0.731765,0.226667,BEhuvaI5BVaR,RICHARD M. DALEY
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.563137,0.261212,0.725098,0.272424,coo8KK7q6A72,ZHANG RONGMAO
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.566275,0.273636,0.666275,0.285152,0P9rVSbeNdB4,SHENYANG
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.526667,0.380303,0.588235,0.394242,1GDArufutI5y,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.628235,0.380606,0.702353,0.394242,QyD751r4fCU1,Shenyang
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.736863,0.411515,0.868235,0.424545,rntIekANI8BO,Zhang Rongmao
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.199216,0.411818,0.34,0.424848,96TaHazXGIM7,Richard M. Daley
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.514902,0.412424,0.580784,0.425758,kbyVj6qhZSPi,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.696471,0.443939,0.774118,0.45697,rJpaMvepsNln,Shenyang
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.353725,0.474545,0.415686,0.489091,PokCVpLQmDki,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,ADDRESS,"(0, 0, 0)",0.407451,0.554545,0.469804,0.568182,HqVr414KRg59,Chicago
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,HANDWRITING,"(0, 0, 0)",0.593725,0.750606,0.811765,0.813636,xdawEv0DUH6P,3h.5.
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.730196,0.819394,0.876471,0.830606,Gghr7ccN6lS2,ZHANG RONGMAO
+C:\Users\spedrickcase\OneDrive - Lambeth Council\Apps\doc_redaction\input/Partnership-Agreement-Toolkit_0_0.pdf_6.png,7,NAME,"(0, 0, 0)",0.34,0.821515,0.501176,0.831515,vOMIv1RS5Sag,RICHARD M. DALEY

doc_redaction/example_data/example_outputs/doubled_output_joined.pdf_ocr_output.csv

@@ -0,0 +1,923 @@
+page,text,left,top,width,height,line
+1,5-Point Networking Email,0.404314,0.050606,0.189804,0.012121,1
+1,"Steve Dalton, the author of The 2-Hour Job Search believes the perfect networking email is a ""5-Point E-mail"". The five",0.058824,0.086061,0.859608,0.012727,2
+1,points are as follows:,0.059216,0.10303,0.152941,0.012727,3
+1,1. 100 words or less,0.088627,0.136667,0.156078,0.010303,4
+1,2. No mention of jobs (in subject or body),0.088235,0.153333,0.31451,0.012727,5
+1,"3. Connection goes first (e.g., ND connection)",0.087843,0.170606,0.341569,0.01303,6
+1,4. Generalize your interest,0.087843,0.187879,0.205098,0.012424,7
+1,5. Maintain control of the follow up,0.088627,0.204545,0.27098,0.012727,8
+1,Here's an example of what a 5-Point email would look like:,0.059608,0.255455,0.42549,0.012727,9
+1,Subject: Notre Dame MBA Student Seeking Your Advice,0.117255,0.289394,0.414118,0.012424,10
+1,"Dear Mr. Jones,",0.118039,0.323939,0.112549,0.011515,11
+1,"My name is Brooke Franklin, and I'm a first-year Notre Dame MBA student who found your",0.118431,0.35697,0.661569,0.01303,12
+1,information in the ND alumni database. May I have 15 minutes of your time to ask you about,0.118039,0.374242,0.677255,0.012727,13
+1,your experience with IBM? I'm trying to learn more about marketing careers at technology,0.117255,0.391212,0.660784,0.01303,14
+1,companies and your insights would be very helpful.,0.117647,0.407879,0.373333,0.01303,15
+1,"I realize this may be a busy time for you, so if we're unable to connect this week, I'll try again",0.118039,0.442121,0.674902,0.012727,16
+1,next week to see whether that is more convenient.,0.118039,0.459091,0.370588,0.010303,17
+1,"Thank you for your time,",0.117255,0.492727,0.179216,0.012727,18
+1,Brooke,0.118431,0.51,0.050588,0.01,19
+1,The most important part of this email may be the follow-up; an email like this allows you to reach out again in a week if,0.058431,0.543333,0.872157,0.01303,20
+1,you haven't heard back without feeling like you're bothering the person at the other end. If you don't hear anything,0.058431,0.560606,0.843922,0.01303,21
+1,"after the second attempt, you can probably cross him/her off your list and move on to the next contact.",0.058824,0.577273,0.755686,0.01303,22
+2,36 Westmoreland Drive,0.705764,0.026796,0.209996,0.011403,1
+2,Newcastle upon Tyne,0.723499,0.04333,0.192664,0.013968,2
+2,NE1 8LT,0.836759,0.059863,0.079807,0.011117,3
+2,Mr Mark Wilson,0.083837,0.076112,0.138251,0.011403,4
+2,UK Health Trust,0.083837,0.09236,0.143087,0.011403,5
+2,18 Whitehall Square,0.084643,0.108609,0.179766,0.013968,6
+2,London,0.083837,0.125428,0.066102,0.011117,7
+2,SW1 9LT,0.083837,0.141391,0.083031,0.011403,8
+2,11th January 2015,0.755744,0.154789,0.161225,0.017389,9
+2,Dear Mr Wilson,0.083837,0.174173,0.137042,0.011403,10
+2,Re: Community Health Development Officer [HD/12/2014],0.083837,0.201539,0.544135,0.014253,11
+2,"I am writing to apply for the above post, as advertised on the Health UK recruitment site. I am",0.08424,0.228905,0.828295,0.014253,12
+2,a sociology graduate with a 2: 1from Newcastle University. I have relevant health awareness,0.083434,0.245439,0.822249,0.014253,13
+2,"experience, and I am looking for a position where I can employ my knowledge and skills in",0.083434,0.261973,0.802499,0.013968,14
+2,support of health and community development. I enclose my CV for your attention.,0.083434,0.277936,0.731963,0.014253,15
+2,I am eager to work for UK Health Trust because of your ground-breaking work within the field,0.08424,0.305302,0.825877,0.014253,16
+2,of community health. I became aware of the work of the Trust when carrying out my,0.083434,0.322121,0.744055,0.013968,17
+2,"dissertation, 'Generational Change in Local Health Awareness, where I researched health",0.083031,0.338084,0.798468,0.014253,18
+2,awareness of children and elderly people in a deprived location. I referred to a number of,0.083031,0.354618,0.792019,0.013968,19
+2,publications produced by UK Health Trust and was impressed by the innovative techniques,0.083837,0.371152,0.809351,0.013968,20
+2,your organisation uses to engage local community members in projects. The Community,0.083031,0.387685,0.788795,0.014253,21
+2,Health Development Officer position would further develop my existing abilities and my,0.08424,0.403934,0.771463,0.014253,22
+2,"understanding of community development, allowing me to contribute in a practical way to",0.083837,0.420468,0.789601,0.013968,23
+2,enhancing the health of disadvantaged people.,0.083434,0.436716,0.415961,0.013968,24
+2,The volunteer development aspect of the position particularly appeals to me. I have worked,0.083031,0.469213,0.811769,0.014538,25
+2,"in the voluntary sector, providing services tackling health inequalities and promoting healthy",0.083837,0.485747,0.814994,0.014253,26
+2,living in Newcastle. I promoted health awareness through one to one sessions and in large,0.083434,0.501995,0.805723,0.014253,27
+2,"groups and developed interpersonal skills, confidence and patience when engaging and",0.083031,0.518529,0.787183,0.014253,28
+2,"motivating participants. While raising the group's profile using social media, the local press",0.083434,0.534778,0.804917,0.013968,29
+2,"and at presentations to youth clubs, faith meetings and care homes I recognised the need to",0.083434,0.551596,0.820637,0.013968,30
+2,"change my delivery style to suit the audience. As a volunteer teacher in Ghana, I developed",0.083434,0.56756,0.8158,0.014253,31
+2,communication and team-building skills essential to your advertised role; liaising with,0.083434,0.584094,0.753325,0.013968,32
+2,colleagues and parents and a lively group of twenty-five 7-8 year olds to arrange a,0.083434,0.600627,0.731963,0.014253,33
+2,"community event. My retail experience, coupled with my extracurricular activities additionally",0.083434,0.617161,0.822249,0.013968,34
+2,"enhanced my ability to develop others, as I was responsible for inducting and training my",0.083434,0.633409,0.79081,0.014253,35
+2,peers.,0.083837,0.652509,0.05401,0.011117,36
+2,"In relation to the fundraising and budgeting aspect of the role, I have experience of raising",0.08424,0.68244,0.798065,0.014253,37
+2,"substantial amounts of money through several successful charity events, including a well -",0.083031,0.698404,0.802096,0.014538,38
+2,attended fashion show. I was also elected Treasurer of NU Sociology Society with,0.083434,0.715222,0.728335,0.014253,39
+2,responsibility for managing a budget of £3000.,0.083434,0.731471,0.411528,0.014538,40
+2,The necessity to travel to identify community issues only adds to the appeal of the position. I,0.083031,0.758837,0.82104,0.014253,41
+2,"enjoy driving, hold a full clean driving licence and I am very interested in relocating to London",0.083434,0.775086,0.828295,0.014538,42
+2,to work for UK Health Trust.,0.083031,0.791619,0.247481,0.011688,43
+2,Thank you for considering my application. I look forward to hearing from you.,0.083434,0.824401,0.68158,0.014253,44
+2,Yours sincerely,0.082628,0.857184,0.138251,0.014253,45
+2,Rachel Sullivan,0.083837,0.889966,0.137042,0.011403,46
+3,SisterCities,0.169804,0.033333,0.238431,0.028182,1
+3,Partnership Agreement,0.516078,0.027879,0.440784,0.032424,2
+3,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788,3
+3,Connect globally. Thrive locally.,0.169804,0.08697,0.238824,0.01303,4
+3,Toolkit,0.830588,0.07303,0.126667,0.025152,5
+3,Types of Affiliations,0.117255,0.157576,0.241961,0.02,6
+3,Sister City Relationship,0.117647,0.187273,0.196863,0.013939,7
+3,"A Sister City relationship is formed when the mayor or highest elected official (or, if elections",0.117255,0.211212,0.738824,0.013636,8
+3,"do not take place, highest appointed official) from a U.S. community and a community in",0.117647,0.227273,0.70902,0.013939,9
+3,another country or territory sign a formal agreement on behalf of their communities endorsing a,0.117647,0.243636,0.761961,0.013636,10
+3,"""sister city/sister cities"" relationship. Sister city agreements shall be considered active/valid",0.118039,0.259697,0.731373,0.013939,11
+3,unless otherwise indicated by one or both of the respective communities.,0.118039,0.276061,0.58549,0.013636,12
+3,Sister Cities International shall formally recognize only those relationships by cities/members in,0.118039,0.299697,0.758824,0.013636,13
+3,good standing (i.e. who are current on membership dues) in its Membership Directory or on its,0.117647,0.316061,0.754902,0.013636,14
+3,"website. However, Sister Cities International shall not assert as invalid or otherwise impugn the",0.116863,0.332121,0.760784,0.013636,15
+3,legitimacy of those relationships formed by non-members.,0.118039,0.348485,0.466275,0.013636,16
+3,Friendship City,0.118039,0.372121,0.127059,0.013939,17
+3,"A Friendship City or Friendship Cities relationship is often formed by cities as a ""stepping",0.117255,0.395758,0.714118,0.013636,18
+3,"stone"" to a more formal ""Sister City"" agreement. Typically Friendship City agreements are",0.117647,0.411515,0.720392,0.014242,19
+3,referred to as such in the formal documents that are signed. Sister Cities International shall,0.118039,0.428182,0.72549,0.013636,20
+3,recognize Friendship City relationships by members in its Membership Directory and website.,0.118039,0.444242,0.747843,0.013636,21
+3,As per Sister Cities International Board of Directors:,0.117255,0.467879,0.413333,0.013636,22
+3,Sister Cities International will recognize a new sister cities affiliation between a,0.169412,0.492121,0.626667,0.013333,23
+3,"U.S. and an international community, even though another affiliation may exist",0.169412,0.507879,0.625098,0.013636,24
+3,"between that international community and a different U.S. community, only if a",0.169412,0.524545,0.62902,0.013636,25
+3,cooperative agreement among all involved communities is filed with Sister Cities,0.16902,0.540606,0.643137,0.013636,26
+3,"International. If a cooperative agreement is denied, or no response to the request",0.170196,0.556667,0.647843,0.013333,27
+3,"is received within a reasonable amount of time, Sister Cities International will",0.169412,0.57303,0.612157,0.012727,28
+3,recognize the partnership as a friendship city and it will be delineated as such,0.169412,0.589091,0.621176,0.013636,29
+3,with a symbol in the membership directories.,0.168627,0.605455,0.358824,0.013333,30
+3,The cooperative agreement must be sent by the Mayor/County,0.168627,0.628788,0.509412,0.013939,31
+3,"Executive/Governor of the requesting community, and must be sent to the",0.169804,0.645152,0.595294,0.014242,32
+3,Mayor/County Executive/Governor of each of the existing partnership,0.169804,0.661212,0.555294,0.013636,33
+3,communities. Although the Mayor/County Executive/Governor may request input,0.16902,0.677879,0.647451,0.013636,34
+3,"from, or may be given input by, the sister cities program, it is up to the discretion",0.168627,0.693939,0.647059,0.013939,35
+3,of the Mayor/County Executive/Governor to sign the cooperative agreement.,0.16902,0.709697,0.612941,0.013939,36
+3,Although Sister Cities International will help with the cooperative agreement,0.168627,0.726364,0.605882,0.013636,37
+3,"process, it is up to the requesting community to get the agreement signed. Sister",0.169412,0.742121,0.650196,0.013939,38
+3,"Cities International will not, in any way, force a community to ""share"" and sign",0.16902,0.758182,0.623922,0.014242,39
+3,the cooperative agreement.,0.168627,0.774848,0.219216,0.013333,40
+3,"To place a relationship into Emeritus status, the mayor or highest elected official of the U.S.",0.117255,0.798485,0.736471,0.013939,41
+3,community must write a letter to the mayor of the foreign city indicating that they wish to,0.118039,0.814545,0.70902,0.013636,42
+3,"remain sister cities, but understand that the relationship will remain inactive until such time as",0.118039,0.831212,0.747451,0.013333,43
+3,both cities are able to sustain an active relationship. Sister Cities International should be,0.118039,0.847273,0.705098,0.013636,44
+3,informed in writing by the mayor of the U.S. city of the situation. Sister Cities International will,0.118039,0.863333,0.746275,0.013636,45
+4,SisterCities,0.169804,0.033333,0.238824,0.028182,1
+4,Partnership Agreement,0.516078,0.027879,0.440784,0.032424,2
+4,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788,3
+4,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303,4
+4,Toolkit,0.83098,0.072727,0.127059,0.025455,5
+4,then place the partnership into Emeritus Status and will reflect this status in directories and all,0.117255,0.132424,0.751373,0.013333,6
+4,lists of sister city programs.,0.118039,0.148788,0.218431,0.013333,7
+4,"If a community wishes to terminate a sister city relationship, then a letter from the mayor or",0.118431,0.172424,0.732549,0.013333,8
+4,highest elected official of the U.S. city should be sent to the mayor of the sister city. Sister,0.118039,0.188485,0.721569,0.013636,9
+4,Cities International should be informed of this action in writing by the mayor of the U.S. city,0.118039,0.204848,0.72902,0.013333,10
+4,and Sister Cities International will then remove the partnership from its directories and all lists,0.117647,0.221212,0.746275,0.013333,11
+4,of sister city programs. We do not recommend terminating a relationship simply because it is,0.117647,0.237273,0.743529,0.013333,12
+4,"dormant. Many partnerships wax and wane over the years, and in many cases a dormant",0.117647,0.253939,0.713333,0.013333,13
+4,partnership may be reinvigorated by local members years after it has been inactive.,0.118039,0.269697,0.664314,0.013636,14
+4,General Guidelines,0.118039,0.295152,0.231765,0.016061,15
+4,In order for a sister city/county/state partnership to be recognized by Sister Cities International,0.118431,0.324242,0.754902,0.013636,16
+4,"(SCI), the two communities must sign formal documents which clearly endorse the link. This",0.118039,0.340606,0.74,0.013636,17
+4,presumes several key items: that the U.S. community is already a member of SCI and has,0.118039,0.35697,0.718039,0.013636,18
+4,followed proper procedures (e.g. passed a city council resolution declaring the intent to twin,0.117255,0.373333,0.737647,0.013636,19
+4,with the specific city); that both communities share a mutual commitment to the relationship;,0.117255,0.389394,0.740784,0.013636,20
+4,and that both have secured the necessary support structure to build a lasting relationship. You,0.117647,0.405455,0.758039,0.013333,21
+4,should check with your local sister city program to see if they have any additional requirements,0.117647,0.421818,0.760784,0.013636,22
+4,before pursuing a sister city relationship.,0.118039,0.437879,0.323137,0.013636,23
+4,"SCI often refers to these agreements as a ""Sister City Agreement"" or ""Memorandum of",0.118039,0.461515,0.696863,0.013939,24
+4,"Understanding."" However, as the following examples show, the actual name and format of",0.118039,0.477576,0.729804,0.013636,25
+4,your documents is left up to you.,0.117255,0.494242,0.262745,0.013636,26
+4,A few things to keep in mind as you draft your agreement:,0.117255,0.517879,0.463137,0.013636,27
+4,"Your agreement can range from the ceremonial, with language focusing on each city's",0.176471,0.542121,0.69098,0.013939,28
+4,"commitment to fostering understanding, cooperation, and mutual benefit to the precise,",0.176471,0.558485,0.701961,0.013333,29
+4,"with particular areas of interest, specific programs/activities, or more concrete goals",0.176078,0.574848,0.673725,0.013636,30
+4,related to anything from numbers of exchanges to economic development.,0.176863,0.591212,0.596863,0.013636,31
+4,"Don't try to include everything you plan to do. Some specifics, like particular areas of",0.177255,0.620303,0.681176,0.013939,32
+4,"interest or participating institutions are good to include. However, there's no need to",0.176471,0.636667,0.675686,0.013636,33
+4,include all the programs you plan to do if it makes the document too lengthy or limits,0.176863,0.652727,0.678824,0.013939,34
+4,the scope of projects. This is a formal document to establish the relationship; specific,0.176078,0.668788,0.684706,0.013636,35
+4,"tasks, responsibilities, or other nuts-and-bolts text related to implementation or",0.176078,0.685455,0.635686,0.013333,36
+4,administration of the partnership can be expressed more fully in a separate,0.176471,0.701212,0.600392,0.013636,37
+4,memorandum between the respective sister city committees. Your partnership,0.177255,0.717576,0.626667,0.013636,38
+4,agreement is a historical document and should not be dated or limited by being aligned,0.176471,0.733636,0.699216,0.013636,39
+4,with very specific tasks.,0.176078,0.750606,0.190196,0.013333,40
+4,Work with your counterparts. Remember that this is signed by both cities. You should,0.176078,0.779697,0.68549,0.013636,41
+4,share drafts of your agreement with your international partners and solicit feedback on,0.176471,0.795758,0.691765,0.013333,42
+4,what they'd like to see in the agreement. Be flexible to cultural or municipal priorities.,0.176471,0.811818,0.679216,0.013939,43
+4,Ask your counterparts to translate the agreement if it is drafted in English. It is,0.176078,0.841515,0.623137,0.013636,44
+4,important for the citizens of your partner community to be able to read and understand,0.176863,0.857576,0.693725,0.013939,45
+4,the commitment their city has made. Have someone in your own community who,0.176078,0.873939,0.649804,0.013636,46
+5,SisterCities,0.169804,0.033333,0.239216,0.028182,1
+5,Partnership Agreement,0.516078,0.027879,0.441176,0.032121,2
+5,INTERNATIONAL,0.170196,0.06697,0.237255,0.008788,3
+5,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303,4
+5,Toolkit,0.83098,0.07303,0.126667,0.025152,5
+5,speaks that language check the foreign-language version to make sure it mirrors what,0.176471,0.132424,0.688235,0.013333,6
+5,you have in your own agreement.,0.176471,0.148788,0.264706,0.013333,7
+5,Keep it to one page. Ceremonial documents such as these partnership agreements,0.176863,0.178485,0.66549,0.013636,8
+5,work best if they can be posted in their entirety.,0.176078,0.194545,0.380392,0.013636,9
+5,Most sister city agreements include some acknowledgement of the founding principles,0.177255,0.224242,0.694902,0.013636,10
+5,"of the sister city movement- to promote peace through mutual respect, understanding,",0.176471,0.240303,0.698431,0.013333,11
+5,and cooperation.,0.176471,0.25697,0.13451,0.013333,12
+5,Consider using official letterhead and/or other embellishments such as city seals or,0.176863,0.286061,0.665882,0.013333,13
+5,logos to reflect your enhance the document. Sister city agreements are often posted at,0.176863,0.302121,0.695686,0.013636,14
+5,city hall or other municipal offices and should reflect their historical importance,0.176471,0.318485,0.630588,0.013333,15
+5,Look at other agreements your city has signed. These agreements may give you an idea,0.177255,0.347879,0.705098,0.013636,16
+5,"of what is acceptable or possible, and they may be in an easily replicable format. If you",0.176471,0.364242,0.695686,0.013636,17
+5,"cannot access older agreements please contact Sister Cities International, we may",0.176863,0.380303,0.663137,0.013636,18
+5,"have them on file, although we do not have copies of all partnership agreements.",0.176863,0.396667,0.64549,0.013636,19
+5,Documents must be signed by the top elected official of both communities.,0.177255,0.426364,0.601569,0.013333,20
+5,"Check with your mayor, city council, town clerk, et al. to make sure that the agreement",0.176863,0.455758,0.694118,0.013636,21
+5,"is OK with them. The mayor is the one putting his or her name on the paper, and you",0.176863,0.471818,0.677255,0.013333,22
+5,don't want to spend time developing an agreement which will never be signed.,0.176863,0.488182,0.629412,0.013636,23
+5,Official documents are usually signed during a formal ceremony recognizing the,0.176863,0.517576,0.638431,0.013636,24
+5,partnership. Be sure both communities receive a signed set of the official documents,0.177255,0.533939,0.683922,0.013636,25
+5,for their records.,0.176078,0.550606,0.131373,0.010606,26
+5,Remember to send your signed agreement to Sister Cities International. After we,0.177255,0.579697,0.645098,0.013636,27
+5,receive your agreement we will post the relationship in the City Directory and make sure,0.176863,0.595758,0.703137,0.013636,28
+5,it is included in our Annual Membership Directory.,0.176863,0.612121,0.398039,0.013333,29
+5,Remember that each city's sister city program is independent and can impose requirements,0.118431,0.640606,0.736471,0.013939,30
+5,"like the establishment of a committee, a review period, sustainability/funding plan, among",0.118039,0.65697,0.715686,0.013636,31
+5,"others, before sanctioning a sister city agreement. Check with your local program or mayor's",0.117647,0.672727,0.743529,0.014242,32
+5,office to see if this is the case.,0.117647,0.689091,0.241176,0.011515,33
+5,On the following pages you'll find a series of partnership agreements to give you an idea of,0.118039,0.717879,0.728627,0.013939,34
+5,"what is possible. While you should feel free to use some of the formatting and language, we",0.117255,0.734242,0.73451,0.013636,35
+5,encourage you to make your agreement your own and be creative with what you produce. If,0.117647,0.750606,0.737647,0.013636,36
+5,you are unsure about your agreement or want advice you can always solicit feedback by,0.117647,0.766667,0.708627,0.013636,37
+5,sending it to our Membership Director at akaplan@sister-cities.org or contacting us at (202),0.117647,0.782727,0.732157,0.013636,38
+5,347-8630.,0.117647,0.799394,0.080392,0.010303,39
+6,SisterCities,0.169412,0.033333,0.239608,0.028485,1
+6,Partnership Agreement,0.516471,0.027879,0.440784,0.032727,2
+6,INTERNATIONAL,0.170196,0.066667,0.238431,0.009091,3
+6,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333,4
+6,Toolkit,0.830588,0.072727,0.127843,0.025758,5
+6,"jull bubzig 2000 3,312",0.378039,0.291212,0.32549,0.019394,6
+6,ABU DHABI MUNICIPALITY & TOWN PLANNING,0.376471,0.316667,0.327451,0.016667,7
+6,AN AGREEMENT FOR THE ESTABLISHMENT OF,0.260784,0.373636,0.52549,0.012727,8
+6,SISTER CITIES RELATIONSHIP,0.337647,0.393636,0.342745,0.012121,9
+6,BETWEEN,0.454902,0.413636,0.110588,0.011212,10
+6,THE CITY OF ABU DHABI ( U. A.E),0.337255,0.432727,0.375686,0.013939,11
+6,AND,0.487843,0.452727,0.048235,0.011212,12
+6,"HOUSTON, TEXAS ( U.S.A)",0.385882,0.471515,0.298039,0.014848,13
+6,"The Sister City Program, administered by Sister Cities International, was initiated",0.221961,0.525455,0.597255,0.01303,14
+6,By the President of the United States of America in 1956 to encourage greater,0.222745,0.539394,0.561961,0.012727,15
+6,Friendship and understanding between the United States and other nations through,0.222745,0.553333,0.608235,0.012727,16
+6,Direct personal contact: and,0.222745,0.567576,0.20549,0.012424,17
+6,"In order to foster those goals, the people of Abu Dhabi and Houston, in a gesture of",0.222353,0.594242,0.603529,0.012424,18
+6,"Friendship and goodwill, agree to collaborate for the mutual benefit of their",0.222745,0.608182,0.547843,0.01303,19
+6,"Communities by exploring education, economic and cultural opportunities.",0.222353,0.622121,0.541961,0.012121,20
+6,"Abu Dhabi and Houston, sharing a common interest in energy, technology and",0.221569,0.648788,0.574118,0.012424,21
+6,"medicine, and the desire to promote mutual understanding among our citizens do",0.222353,0.66303,0.588235,0.012121,22
+6,"hereby proclaim themselves Sister Cities beginning on the 13th day of March 2001,",0.221961,0.673636,0.594118,0.015758,23
+6,the date of Houston City Council resolution estatblishing the Sister City,0.221961,0.690303,0.519608,0.01303,24
+6,relationship became effective.,0.221569,0.705152,0.217647,0.012424,25
+6,"Signed on this 26 of October 2002, in duplicate in the Arabic and English",0.221569,0.732121,0.533333,0.01303,26
+6,"Languages, both text being equally authentic.",0.221961,0.746667,0.328627,0.012727,27
+6,A,0.344314,0.768485,0.084706,0.030303,28
+6,Sheikh Mohammed bin Butti AI Hamed,0.245882,0.806364,0.366275,0.010909,29
+6,Lee P.Brown,0.729412,0.806364,0.118824,0.010303,30
+6,Chairman of Abu Dhabi Municipality,0.24549,0.823636,0.342353,0.012727,31
+6,Mayor of Houston,0.704706,0.823333,0.166667,0.012424,32
+6,&Town Planning,0.324314,0.841212,0.155686,0.012424,33
+7,SisterCities,0.169412,0.033333,0.239608,0.028485,1
+7,Partnership Agreement,0.516078,0.027879,0.441176,0.032424,2
+7,INTERNATIONAL,0.17098,0.066667,0.237255,0.009091,3
+7,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333,4
+7,Toolkit,0.83098,0.072727,0.127059,0.025758,5
+7,THE CITY OF NEW YORK,0.438824,0.262121,0.240784,0.009697,6
+7,OFFICE OF THE MAYOR,0.450196,0.27697,0.220392,0.009697,7
+7,"NEW YORK, N.Y. 10007",0.461176,0.29303,0.196863,0.010303,8
+7,THE NEW YORK CITY-LONDON SISTER CITY PARTNERSHIP,0.267451,0.355758,0.582745,0.011818,9
+7,Memorandum of Understanding,0.420392,0.371212,0.274902,0.013333,10
+7,The Sister City partnership between New York City and London will foster mutually,0.201176,0.402121,0.674118,0.014242,11
+7,beneficial solutions to common challenges for these two great cosmopolitan entities.,0.201176,0.417273,0.66902,0.013636,12
+7,"Consequently, the Sister City relationship between the two will be one of the most",0.201176,0.432727,0.652549,0.015152,13
+7,"important in their network of global partnerships, as it strives to:",0.201176,0.448182,0.50902,0.015455,14
+7,Encourage and publicize existing exchanges between London and New York City so,0.230588,0.480303,0.671373,0.015152,15
+7,that they can flourish to benefit a wider cross-section of the citizens of both;,0.230588,0.496061,0.602353,0.015152,16
+7,"Support and promote the development of new social, economic, academic and",0.230196,0.512424,0.618431,0.015455,17
+7,community programs to encourage both cities' citizens to share their experiences as a,0.229804,0.527879,0.678039,0.014848,18
+7,medium for learning from one another;,0.229804,0.543636,0.309412,0.013939,19
+7,Generate an improvement of the operation of the cities' various government agencies,0.229804,0.56,0.676078,0.014545,20
+7,by serving as a conduit of information;,0.22902,0.575758,0.307843,0.014848,21
+7,"Identify themes, common to both, that can generate new initiatives to further and",0.229412,0.591818,0.640784,0.015152,22
+7,"nurture the increasingly powerful financial, social and cultural relationships between",0.22902,0.607576,0.671373,0.014242,23
+7,the cities;,0.22902,0.624545,0.076471,0.012424,24
+7,Promote key mayoral priorities relevant to both London and New York City;,0.228627,0.639394,0.608627,0.015152,25
+7,Provide financial or in kind support to community-led programs that advance the,0.228627,0.656061,0.641569,0.013636,26
+7,aims of the Sister City partnership;,0.22902,0.672121,0.275294,0.013636,27
+7,"With the above purposes in mind, the Mayor of the City of New York and the Mayor of",0.198824,0.702424,0.697647,0.014848,28
+7,London solemnly confirm that these two cities are united by an official partnership by the,0.198824,0.718182,0.710196,0.014545,29
+7,protocol of this Memorandum of Understanding.,0.198431,0.733939,0.384314,0.015152,30
+7,This agreement will go into effect from the date of signatures.,0.310196,0.780606,0.488235,0.014545,31
+7,Signed in March of 2001,0.455686,0.796364,0.19451,0.013636,32
+7,Thedder Rudolph W. Giuliani,0.178824,0.795455,0.244314,0.100909,33
+7,Mayor,0.311373,0.894848,0.053333,0.012727,34
+7,Ken Mayor Livingstone,0.672157,0.877576,0.132941,0.029091,35
+7,New York City,0.287843,0.909091,0.121176,0.013333,36
+7,London,0.701961,0.909091,0.061569,0.010606,37
+8,SisterCities,0.169412,0.03303,0.24,0.028182,1
+8,Partnership Agreement,0.515686,0.027576,0.441961,0.03303,2
+8,INTERNATIONAL,0.169804,0.066667,0.238431,0.009091,3
+8,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333,4
+8,Toolkit,0.83098,0.072727,0.127451,0.025758,5
+8,CHIC OF STATE,0.247451,0.190606,0.141961,0.036364,6
+8,City of Long Beach,0.388627,0.196667,0.476471,0.066364,7
+8,California,0.551373,0.257273,0.136471,0.033333,8
+8,Sister City Agreement,0.321961,0.305455,0.378431,0.035152,9
+8,between the,0.464706,0.352727,0.084314,0.009697,10
+8,City of Long Beach,0.38,0.378485,0.252549,0.01697,11
+8,"California, USA",0.4,0.397576,0.21098,0.016061,12
+8,and the,0.48,0.415152,0.053333,0.009091,13
+8,City of San Pablo de Manta,0.321569,0.428788,0.369804,0.01697,14
+8,"Ecuador, South America",0.347451,0.447879,0.317255,0.015152,15
+8,"In accordance with the authorization and approval expressed by the City of Long Beach,",0.261569,0.482121,0.536863,0.012121,16
+8,"California, USA, and the City of San Pablo de Manta, Ecundor, South America, it is declared",0.217647,0.492727,0.581176,0.01303,17
+8,"that a ""Sister City Agreement between the two cities is hereby established for the following",0.217647,0.502727,0.581569,0.012121,18
+8,purposes:,0.216863,0.516061,0.058039,0.009394,19
+8,(1) to promote and expand the effective and mutually beneficial cooperation between,0.278824,0.532727,0.520392,0.012424,20
+8,the people of Long Beach and the people of San Pablo de Manta; and,0.218039,0.543636,0.40549,0.012424,21
+8,"(2) to promote international goodwill, understanding, and expanded business",0.279216,0.56303,0.520784,0.012424,22
+8,"relations between the two cities and their respective nations by the exchange of people, ideas, and",0.218039,0.573636,0.581569,0.012121,23
+8,"information in a unide variety of economic, social, cultural, municipal, environmental,",0.218039,0.584242,0.581176,0.012121,24
+8,"professional, technical, youth, and other endeavors; and",0.217647,0.594848,0.333333,0.012121,25
+8,"(3) to foster and encourage charitable, scientific, trade and commerce, literary and",0.279608,0.613939,0.520784,0.012727,26
+8,educational activities between the two cities;,0.218039,0.625455,0.265882,0.009697,27
+8,This Sister City Agreement shall be officially established and shall become effective when,0.263137,0.644545,0.536863,0.012727,28
+8,"this document has been duly executed by the Mayor of Long Beach, California, USA, and the",0.218824,0.654848,0.581961,0.012424,29
+8,"Mayor of San Pablo de Manta, Ecundor, South America.",0.218431,0.665758,0.338824,0.012121,30
+8,STATE OFFICE,0.276471,0.713636,0.050588,0.048788,31
+8,Beverly 0 Neill,0.587451,0.736667,0.121961,0.013636,32
+8,"Mayor, City of Long Beach",0.542353,0.751212,0.21098,0.013636,33
+8,"California, USA",0.582745,0.765758,0.125098,0.01303,34
+8,10.2aulus,0.490588,0.771818,0.220392,0.062424,35
+8,Ing. Jorge O. Zambrano Cedeño,0.527059,0.825152,0.242745,0.013333,36
+8,"Mayor, City of San Pablo de Manta",0.505098,0.839394,0.277647,0.013636,37
+8,"Ecuador, South America",0.551765,0.854242,0.188235,0.011818,38
+8,"Dated: September 19, 2000",0.544706,0.883333,0.202745,0.01303,39
+9,SisterCities,0.169412,0.03303,0.24,0.028485,1
+9,Partnership Agreement,0.516078,0.027879,0.441176,0.032424,2
+9,INTERNATIONAL,0.170196,0.066667,0.237647,0.009091,3
+9,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333,4
+9,Toolkit,0.83098,0.072727,0.127451,0.025758,5
+9,REAFFIRMATION OF SISTER CITIES DECLARATION,0.324706,0.165152,0.483529,0.013939,6
+9,adopted by,0.2,0.213333,0.080392,0.013636,7
+9,THE HONORABLE RICHARD M. DALEY,0.396078,0.214242,0.335686,0.012424,8
+9,MAYOR OF CHICAGO,0.472549,0.231212,0.18549,0.011515,9
+9,and,0.199608,0.260909,0.026275,0.010606,10
+9,THE HONORABLE ZHANG RONGMAO,0.401961,0.261212,0.323137,0.011212,11
+9,MAYOR OF SHENYANG,0.463529,0.273636,0.202353,0.011212,12
+9,ON,0.551765,0.298182,0.026667,0.011515,13
+9,"JUNE 5, 1995",0.500392,0.323636,0.128235,0.014848,14
+9,"On this the tenth anniversary of the signing of a sister city agreement, in order to further",0.255686,0.36303,0.67098,0.015152,15
+9,the traditional links of friendship between Chicago and Shenyang and to reaffirm their mutual,0.198824,0.378788,0.727843,0.015455,16
+9,"aspiration to work in unison for the benefit of their cities and nations, the Honorable Mayor",0.199608,0.394848,0.727843,0.014848,17
+9,"Richard M. Daley, Mayor of the City of Chicago, and the Honorable Zhang Rongmao, Mayor",0.199216,0.411212,0.727451,0.014242,18
+9,"of the City of Shenyang, on this fifth day of June 1995, do hereby acknowledge and reaffirm the",0.199216,0.42697,0.72549,0.014848,19
+9,sister cities agreement between the City of Chicago and the City of Shenyang.,0.199608,0.443636,0.57451,0.014242,20
+9,"The City of Chicago and the City of Shenyang on the basis of friendly cooperation,",0.256078,0.473939,0.665098,0.015152,21
+9,equality and mutual benefit will continue to develop a sister cities relationship to promote and,0.2,0.490303,0.724706,0.014242,22
+9,broaden economic cooperation and cultural exchanges between the two cities.,0.199216,0.506061,0.57451,0.014242,23
+9,The two cities do hereby declare their interest in exploring the establishment of business,0.255294,0.537273,0.668235,0.015455,24
+9,and trade relations between Chicago and Shenyang.,0.198824,0.554545,0.387843,0.013636,25
+9,"In addition, exchanges will be promoted in the area of the arts such as exhibits, music,",0.254118,0.583939,0.666667,0.015455,26
+9,dance and other cultural activities.,0.198431,0.601212,0.256471,0.010606,27
+9,"In addition, exchanges will be promoted in education and the establishment of contacts",0.254118,0.630303,0.668627,0.015758,28
+9,within educational institutions encouraged.,0.198824,0.647273,0.32,0.014242,29
+9,"In addition, we declare our intention to promote exchanges in such fields as science and",0.253725,0.678182,0.668627,0.014848,30
+9,"technology, sports, health, youth and any areas that will contribute to the prosperity and the",0.198039,0.693636,0.722745,0.015152,31
+9,further development of friendship between the people of our two cities.,0.194902,0.711515,0.525098,0.013636,32
+9,3h.5.,0.593725,0.750606,0.218039,0.06303,33
+9,THE HONORABLE RICHARD M. DALEY,0.197255,0.821515,0.303529,0.010606,34
+9,THE HONORABLE ZHANG RONGMAO,0.588627,0.819394,0.287843,0.011818,35
+9,MAYOR OF CHICAGO,0.195686,0.835758,0.164706,0.010606,36
+9,MAYOR OF SHENYANG,0.587451,0.835455,0.177647,0.010303,37
+10,Skills_based_CV.qxd 5/8/11 3:55 pm Page,0.17777,0.135381,0.308796,0.008545,1
+10,agcas,0.726169,0.191722,0.053368,0.011749,2
+10,Example of a skills-based CV,0.3894,0.205874,0.224144,0.011482,3
+10,ASHLEY GILL,0.459698,0.246195,0.082812,0.008278,4
+10,3 Lappage Court,0.2212,0.259012,0.080972,0.008545,5
+10,Telephone: 01882 652349,0.592565,0.259012,0.129555,0.008278,6
+10,"Tyler Green, Bucks.",0.220464,0.269159,0.092381,0.008278,7
+10,Mobile: 07717 121824,0.593669,0.269159,0.112992,0.006676,8
+10,HP8 4JD,0.2212,0.279306,0.040486,0.006409,9
+10,Email: ashleygill2023@gotmail.com,0.594038,0.279039,0.178874,0.008545,10
+10,Personal Details,0.221568,0.299332,0.095326,0.007744,11
+10,Summary,0.220832,0.321495,0.048215,0.008278,12
+10,Business studies with Spanish undergraduate.,0.273463,0.340988,0.229297,0.008812,13
+10,Ability to speak French and Spanish.,0.272727,0.351135,0.179242,0.008545,14
+10,Extensive business experience including an internship with Top Choice Holidays.,0.273095,0.361015,0.398233,0.008812,15
+10,Education And Qualifications,0.2212,0.381041,0.144277,0.008278,16
+10,2008 present,0.220832,0.401602,0.074715,0.008011,17
+10,Buckinghamshire Edge University,0.386824,0.401068,0.167096,0.008545,18
+10,BA International Business Studies with Spanish (expected 2:1),0.386824,0.410681,0.308796,0.008812,19
+10,Relate your degree to,0.230033,0.420027,0.100847,0.008278,20
+10,Study semester at The University of Valloid (Spain).,0.399338,0.420828,0.252852,0.008812,21
+10,the job by listing your,0.229665,0.429105,0.101583,0.008278,22
+10,Six-month work placement in Madrid.,0.399338,0.431242,0.188811,0.008545,23
+10,relevant modules/,0.230033,0.438718,0.085388,0.007744,24
+10,Relevant modules included: Business Planning; Sales Promotion and,0.399338,0.441389,0.338241,0.008545,25
+10,dissertation.,0.230033,0.448064,0.057784,0.006676,26
+10,Marketing; and Business Operations Management.,0.398969,0.451268,0.25322,0.008812,27
+10,2000 2007,0.2212,0.467824,0.061833,0.006409,28
+10,Freebridge School,0.386824,0.46729,0.087965,0.008545,29
+10,"A-Levels: Business Studies (B), French (C)",0.386088,0.476903,0.200221,0.008812,30
+10,"8 GCSEs including Maths, English, Spanish and French",0.386824,0.487583,0.266838,0.008545,31
+10,Work History,0.220832,0.509212,0.065513,0.008278,32
+10,2008 2011,0.220832,0.529506,0.061833,0.006409,33
+10,Buckinghamshire Edge University Librarian/tour guide,0.386824,0.528972,0.277144,0.008812,34
+10,General administrative and customer service roles.,0.399338,0.539119,0.25138,0.006676,35
+10,Briefly list,0.707766,0.536716,0.045639,0.008011,36
+10,your relevant,0.70703,0.546061,0.061465,0.008011,37
+10,2011 (Feb-Aug),0.2212,0.55514,0.078027,0.008812,38
+10,Audigest S.A. (Madrid) - Audit Assistant,0.386456,0.554873,0.199485,0.009079,39
+10,duties.,0.707398,0.555674,0.030916,0.006409,40
+10,Six months' work experience in an international bank.,0.399338,0.565287,0.267575,0.008545,41
+10,Liaising with colleagues and clients in English and Spanish.,0.399338,0.575434,0.292602,0.008545,42
+10,2010 (June-Dec),0.220832,0.591188,0.082444,0.008278,43
+10,Finsbury's supermarket (Hazelbridge) — Supervisor,0.386824,0.591188,0.250644,0.008812,44
+10,Managing a small team.,0.398969,0.601602,0.121089,0.008545,45
+10,Customer service in a busy competitive environment.,0.398969,0.611215,0.264262,0.008545,46
+10,2010 (Jan-Aug),0.2212,0.627236,0.077291,0.008812,47
+10,Top Choice Holidays and Flights Ltd (Low Wycombe),0.386088,0.627503,0.257637,0.008812,48
+10,Financial Assistant/Supervisor,0.386824,0.637383,0.15127,0.008812,49
+10,Working in a range of teams to manage complex financial processes.,0.398969,0.64753,0.341921,0.008812,50
+10,2007 (Jul-Aug),0.220832,0.663284,0.074347,0.008812,51
+10,Dogs Protection League - General Assistant,0.386824,0.663818,0.216783,0.008812,52
+10,Dealing with enquiries and selling packages to a range of clients.,0.399706,0.673431,0.321678,0.009079,53
+10,2006 (Jan-Dec),0.220832,0.689453,0.076187,0.009079,54
+10,McHenry's Restaurant (Low Wycombe) - Supervisor,0.386456,0.68972,0.256533,0.009079,55
+10,Voluntary Experience,0.220464,0.708411,0.106367,0.008545,56
+10,2007/2011,0.220832,0.728438,0.055208,0.008011,57
+10,Teaching English in Mexico/Spain,0.386088,0.727904,0.167832,0.009079,58
+10,Interests,0.2212,0.748465,0.043062,0.006676,59
+10,Active member of University Business Club — Winner of the 'Bucks Best Business Pitch' award in 2010 Enterprise,0.220464,0.768224,0.556864,0.009079,60
+10,"week, judged by Michael Eavis.",0.220464,0.778104,0.15311,0.008812,61
+11,Skills_based_CV.qxd 5/8/11 3:55 pm Page,0.17777,0.135381,0.308428,0.008545,1
+11,Make sure you carefully assess,0.468531,0.23498,0.142068,0.008011,2
+11,Skills And Achievements,0.220832,0.245394,0.121457,0.006676,3
+11,the job advert/job description,0.468163,0.244326,0.139124,0.008278,4
+11,and address all the skills they,0.468531,0.253672,0.13618,0.008278,5
+11,Effective communication,0.2212,0.265421,0.123298,0.006676,6
+11,require.,0.468531,0.263017,0.034965,0.008011,7
+11,"Able to communicate effectively with a wide range of clients and colleagues, by showing interest, carefully",0.233714,0.275567,0.530364,0.008545,8
+11,"listening to needs and appropriately adjusting my message, as demonstrated during my time at Finsbury's",0.23445,0.285447,0.528892,0.008812,9
+11,Supermarket.,0.234082,0.295861,0.066618,0.008278,10
+11,Strong presentation skills and confidence demonstrated by experience of delivering presentations in different,0.23445,0.305474,0.543614,0.008812,11
+11,languages to groups of five to fifty.,0.234082,0.315621,0.172617,0.008812,12
+11,Customer service,0.220832,0.335915,0.085388,0.006676,13
+11,Ability to quickly build rapport with customers and calmly deal with any problems as shown during my retail,0.233714,0.345527,0.541038,0.008812,14
+11,experience in high pressure environments.,0.234082,0.355941,0.210526,0.008278,15
+11,"Capacity to maintain professional relationships through email and other written correspondence, for example,",0.234082,0.365554,0.548767,0.008812,16
+11,"at Audigest in Madrid, where I built longstanding business relationships with customers and colleagues across",0.233714,0.375701,0.549871,0.008812,17
+11,the globe.,0.233714,0.385848,0.049687,0.008278,18
+11,Teamwork,0.220464,0.406142,0.052632,0.006409,19
+11,"At Top Choice Holidays demonstrated excellent teamwork skills in a busy financial environment, such as an",0.233346,0.415754,0.532573,0.008812,20
+11,"ability to listen to clients and managers, perform my role to a high level and support colleagues, resulting in",0.234082,0.425634,0.535885,0.008812,21
+11,promotion.,0.234082,0.436048,0.05484,0.008545,22
+11,Administration,0.220464,0.456075,0.075083,0.006409,23
+11,Prove you have each of the,0.639676,0.453672,0.123666,0.008278,24
+11,"Excellent ability to plan ahead and manage time effectively, for example,",0.23445,0.465688,0.360692,0.008812,25
+11,skills required by outlining,0.63894,0.463017,0.12293,0.008278,26
+11,managing complex roles during my internship at Top Choice Holidays.,0.23445,0.476101,0.346338,0.008545,27
+11,where you performed them,0.63894,0.472363,0.128082,0.008278,28
+11,Gathered data from a wide range of sources during my dissertation,0.234082,0.485714,0.334928,0.008812,29
+11,and how you performed,0.639308,0.481709,0.111888,0.008278,30
+11,them well.,0.63894,0.491055,0.048951,0.006409,31
+11,"whilst balancing my other studies and two jobs, resulting in a 73% grade.",0.233346,0.495861,0.365109,0.008812,32
+11,Experience of travellers' needs,0.2212,0.515888,0.150534,0.008545,33
+11,Recent travel consultancy experience gives me an in-depth understanding of the expectations of holiday,0.23445,0.525768,0.518955,0.008812,34
+11,customers and the competitive nature of the industry.,0.234082,0.535915,0.269047,0.008812,35
+11,International travel experience and language ability give me an empathy with travellers and a passion for,0.234082,0.545794,0.524107,0.008812,36
+11,helping them find a unique holiday experience.,0.234082,0.555941,0.23445,0.008812,37
+11,Initiative,0.2212,0.576235,0.044166,0.006676,38
+11,Self-funding an evening course in bookkeeping during my first accountancy role demonstrated my ability to,0.234082,0.585848,0.535149,0.008812,39
+11,plan ahead and take control of my career.,0.23445,0.595995,0.205006,0.008545,40
+11,Successful study and work in Spain and Mexico show that I can creatively develop my skills and experience and,0.234082,0.605874,0.551711,0.008545,41
+11,adapt to new and different environments.,0.234082,0.616288,0.208686,0.008278,42
+11,Sales knowledge,0.220464,0.636315,0.083916,0.008011,43
+11,Wide experience of financial roles gives me an awareness of the tight monetary pressures which drive UK,0.234082,0.645928,0.525212,0.009346,44
+11,service industries.,0.234082,0.656609,0.088333,0.006943,45
+11,Raised sales at The Dogs Protection League by 12% by up selling add-on packages to new and existing,0.23445,0.665955,0.505705,0.009079,46
+11,customers.,0.234082,0.67717,0.054472,0.006142,47
+11,Language ability,0.2212,0.696395,0.082444,0.008812,48
+11,"Spanish fluency obtained working overseas, French semi-fluent.",0.233714,0.706008,0.323151,0.009079,49
+11,Referees,0.2212,0.726569,0.041958,0.006676,50
+11,Include all your referee details including their email and,0.351859,0.722029,0.259109,0.008545,51
+11,phone number (but ask for their permission first).,0.352227,0.731108,0.230401,0.008545,52
+11,"Professional: Mr. Jose Andreas, Management Accountant, Audigest, Avenida de Concha Espina 2, Madrid, ES-",0.2212,0.746328,0.537725,0.008812,53
+11,"28036, +34 91 398 5476, j.andreas@audigest.es",0.2212,0.756475,0.238498,0.008278,54
+11,"Academic: Dr. Jane Luffle, Personal Tutor, Buckinghamshire Edge University, Due Road, Low Wycombe, Bucks,",0.220464,0.776502,0.536621,0.008812,55
+11,"HD15 3DL, 01628 435 6784, j.luffle@bedge.ac.uk",0.2212,0.786382,0.244755,0.008545,56
+12,5-Point Networking Email,0.404314,0.050606,0.189804,0.012121,1
+12,"Steve Dalton, the author of The 2-Hour Job Search believes the perfect networking email is a ""5-Point E-mail"". The five",0.058824,0.086061,0.859608,0.012727,2
+12,points are as follows:,0.059216,0.10303,0.152941,0.012727,3
+12,1. 100 words or less,0.088627,0.136667,0.156078,0.010303,4
+12,2. No mention of jobs (in subject or body),0.088235,0.153333,0.31451,0.012727,5
+12,"3. Connection goes first (e.g., ND connection)",0.087843,0.170606,0.341569,0.01303,6
+12,4. Generalize your interest,0.087843,0.187879,0.205098,0.012424,7
+12,5. Maintain control of the follow up,0.088627,0.204545,0.27098,0.012727,8
+12,Here's an example of what a 5-Point email would look like:,0.059608,0.255455,0.42549,0.012727,9
+12,Subject: Notre Dame MBA Student Seeking Your Advice,0.117255,0.289394,0.414118,0.012424,10
+12,"Dear Mr. Jones,",0.118039,0.323939,0.112549,0.011515,11
+12,"My name is Brooke Franklin, and I'm a first-year Notre Dame MBA student who found your",0.118431,0.35697,0.661569,0.01303,12
+12,information in the ND alumni database. May I have 15 minutes of your time to ask you about,0.118039,0.374242,0.677255,0.012727,13
+12,your experience with IBM? I'm trying to learn more about marketing careers at technology,0.117255,0.391212,0.660784,0.01303,14
+12,companies and your insights would be very helpful.,0.117647,0.407879,0.373333,0.01303,15
+12,"I realize this may be a busy time for you, so if we're unable to connect this week, I'll try again",0.118039,0.442121,0.674902,0.012727,16
+12,next week to see whether that is more convenient.,0.118039,0.459091,0.370588,0.010303,17
+12,"Thank you for your time,",0.117255,0.492727,0.179216,0.012727,18
+12,Brooke,0.118431,0.51,0.050588,0.01,19
+12,The most important part of this email may be the follow-up; an email like this allows you to reach out again in a week if,0.058431,0.543333,0.872157,0.01303,20
+12,you haven't heard back without feeling like you're bothering the person at the other end. If you don't hear anything,0.058431,0.560606,0.843922,0.01303,21
+12,"after the second attempt, you can probably cross him/her off your list and move on to the next contact.",0.058824,0.577273,0.755686,0.01303,22
+13,36 Westmoreland Drive,0.705764,0.026796,0.209996,0.011403,1
+13,Newcastle upon Tyne,0.723499,0.04333,0.192664,0.013968,2
+13,NE1 8LT,0.836759,0.059863,0.079807,0.011117,3
+13,Mr Mark Wilson,0.083837,0.076112,0.138251,0.011403,4
+13,UK Health Trust,0.083837,0.09236,0.143087,0.011403,5
+13,18 Whitehall Square,0.084643,0.108609,0.179766,0.013968,6
+13,London,0.083837,0.125428,0.066102,0.011117,7
+13,SW1 9LT,0.083837,0.141391,0.083031,0.011403,8
+13,11th January 2015,0.755744,0.154789,0.161225,0.017389,9
+13,Dear Mr Wilson,0.083837,0.174173,0.137042,0.011403,10
+13,Re: Community Health Development Officer [HD/12/2014],0.083837,0.201539,0.544135,0.014253,11
+13,"I am writing to apply for the above post, as advertised on the Health UK recruitment site. I am",0.08424,0.228905,0.828295,0.014253,12
+13,a sociology graduate with a 2: 1from Newcastle University. I have relevant health awareness,0.083434,0.245439,0.822249,0.014253,13
+13,"experience, and I am looking for a position where I can employ my knowledge and skills in",0.083434,0.261973,0.802499,0.013968,14
+13,support of health and community development. I enclose my CV for your attention.,0.083434,0.277936,0.731963,0.014253,15
+13,I am eager to work for UK Health Trust because of your ground-breaking work within the field,0.08424,0.305302,0.825877,0.014253,16
+13,of community health. I became aware of the work of the Trust when carrying out my,0.083434,0.322121,0.744055,0.013968,17
+13,"dissertation, 'Generational Change in Local Health Awareness, where I researched health",0.083031,0.338084,0.798468,0.014253,18
+13,awareness of children and elderly people in a deprived location. I referred to a number of,0.083031,0.354618,0.792019,0.013968,19
+13,publications produced by UK Health Trust and was impressed by the innovative techniques,0.083837,0.371152,0.809351,0.013968,20
+13,your organisation uses to engage local community members in projects. The Community,0.083031,0.387685,0.788795,0.014253,21
+13,Health Development Officer position would further develop my existing abilities and my,0.08424,0.403934,0.771463,0.014253,22
+13,"understanding of community development, allowing me to contribute in a practical way to",0.083837,0.420468,0.789601,0.013968,23
+13,enhancing the health of disadvantaged people.,0.083434,0.436716,0.415961,0.013968,24
+13,The volunteer development aspect of the position particularly appeals to me. I have worked,0.083031,0.469213,0.811769,0.014538,25
+13,"in the voluntary sector, providing services tackling health inequalities and promoting healthy",0.083837,0.485747,0.814994,0.014253,26
+13,living in Newcastle. I promoted health awareness through one to one sessions and in large,0.083434,0.501995,0.805723,0.014253,27
+13,"groups and developed interpersonal skills, confidence and patience when engaging and",0.083031,0.518529,0.787183,0.014253,28
+13,"motivating participants. While raising the group's profile using social media, the local press",0.083434,0.534778,0.804917,0.013968,29
+13,"and at presentations to youth clubs, faith meetings and care homes I recognised the need to",0.083434,0.551596,0.820637,0.013968,30
+13,"change my delivery style to suit the audience. As a volunteer teacher in Ghana, I developed",0.083434,0.56756,0.8158,0.014253,31
+13,communication and team-building skills essential to your advertised role; liaising with,0.083434,0.584094,0.753325,0.013968,32
+13,colleagues and parents and a lively group of twenty-five 7-8 year olds to arrange a,0.083434,0.600627,0.731963,0.014253,33
+13,"community event. My retail experience, coupled with my extracurricular activities additionally",0.083434,0.617161,0.822249,0.013968,34
+13,"enhanced my ability to develop others, as I was responsible for inducting and training my",0.083434,0.633409,0.79081,0.014253,35
+13,peers.,0.083837,0.652509,0.05401,0.011117,36
+13,"In relation to the fundraising and budgeting aspect of the role, I have experience of raising",0.08424,0.68244,0.798065,0.014253,37
+13,"substantial amounts of money through several successful charity events, including a well -",0.083031,0.698404,0.802096,0.014538,38
+13,attended fashion show. I was also elected Treasurer of NU Sociology Society with,0.083434,0.715222,0.728335,0.014253,39
+13,responsibility for managing a budget of £3000.,0.083434,0.731471,0.411528,0.014538,40
+13,The necessity to travel to identify community issues only adds to the appeal of the position. I,0.083031,0.758837,0.82104,0.014253,41
+13,"enjoy driving, hold a full clean driving licence and I am very interested in relocating to London",0.083434,0.775086,0.828295,0.014538,42
+13,to work for UK Health Trust.,0.083031,0.791619,0.247481,0.011688,43
+13,Thank you for considering my application. I look forward to hearing from you.,0.083434,0.824401,0.68158,0.014253,44
+13,Yours sincerely,0.082628,0.857184,0.138251,0.014253,45
+13,Rachel Sullivan,0.083837,0.889966,0.137042,0.011403,46
+14,SisterCities,0.169804,0.033333,0.238431,0.028182,1
+14,Partnership Agreement,0.516078,0.027879,0.440784,0.032424,2
+14,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788,3
+14,Connect globally. Thrive locally.,0.169804,0.08697,0.238824,0.01303,4
+14,Toolkit,0.830588,0.07303,0.126667,0.025152,5
+14,Types of Affiliations,0.117255,0.157576,0.241961,0.02,6
+14,Sister City Relationship,0.117647,0.187273,0.196863,0.013939,7
+14,"A Sister City relationship is formed when the mayor or highest elected official (or, if elections",0.117255,0.211212,0.738824,0.013636,8
+14,"do not take place, highest appointed official) from a U.S. community and a community in",0.117647,0.227273,0.70902,0.013939,9
+14,another country or territory sign a formal agreement on behalf of their communities endorsing a,0.117647,0.243636,0.761961,0.013636,10
+14,"""sister city/sister cities"" relationship. Sister city agreements shall be considered active/valid",0.118039,0.259697,0.731373,0.013939,11
+14,unless otherwise indicated by one or both of the respective communities.,0.118039,0.276061,0.58549,0.013636,12
+14,Sister Cities International shall formally recognize only those relationships by cities/members in,0.118039,0.299697,0.758824,0.013636,13
+14,good standing (i.e. who are current on membership dues) in its Membership Directory or on its,0.117647,0.316061,0.754902,0.013636,14
+14,"website. However, Sister Cities International shall not assert as invalid or otherwise impugn the",0.116863,0.332121,0.760784,0.013636,15
+14,legitimacy of those relationships formed by non-members.,0.118039,0.348485,0.466275,0.013636,16
+14,Friendship City,0.118039,0.372121,0.127059,0.013939,17
+14,"A Friendship City or Friendship Cities relationship is often formed by cities as a ""stepping",0.117255,0.395758,0.714118,0.013636,18
+14,"stone"" to a more formal ""Sister City"" agreement. Typically Friendship City agreements are",0.117647,0.411515,0.720392,0.014242,19
+14,referred to as such in the formal documents that are signed. Sister Cities International shall,0.118039,0.428182,0.72549,0.013636,20
+14,recognize Friendship City relationships by members in its Membership Directory and website.,0.118039,0.444242,0.747843,0.013636,21
+14,As per Sister Cities International Board of Directors:,0.117255,0.467879,0.413333,0.013636,22
+14,Sister Cities International will recognize a new sister cities affiliation between a,0.169412,0.492121,0.626667,0.013333,23
+14,"U.S. and an international community, even though another affiliation may exist",0.169412,0.507879,0.625098,0.013636,24
+14,"between that international community and a different U.S. community, only if a",0.169412,0.524545,0.62902,0.013636,25
+14,cooperative agreement among all involved communities is filed with Sister Cities,0.16902,0.540606,0.643137,0.013636,26
+14,"International. If a cooperative agreement is denied, or no response to the request",0.170196,0.556667,0.647843,0.013333,27
+14,"is received within a reasonable amount of time, Sister Cities International will",0.169412,0.57303,0.612157,0.012727,28
+14,recognize the partnership as a friendship city and it will be delineated as such,0.169412,0.589091,0.621176,0.013636,29
+14,with a symbol in the membership directories.,0.168627,0.605455,0.358824,0.013333,30
+14,The cooperative agreement must be sent by the Mayor/County,0.168627,0.628788,0.509412,0.013939,31
+14,"Executive/Governor of the requesting community, and must be sent to the",0.169804,0.645152,0.595294,0.014242,32
+14,Mayor/County Executive/Governor of each of the existing partnership,0.169804,0.661212,0.555294,0.013636,33
+14,communities. Although the Mayor/County Executive/Governor may request input,0.16902,0.677879,0.647451,0.013636,34
+14,"from, or may be given input by, the sister cities program, it is up to the discretion",0.168627,0.693939,0.647059,0.013939,35
+14,of the Mayor/County Executive/Governor to sign the cooperative agreement.,0.16902,0.709697,0.612941,0.013939,36
+14,Although Sister Cities International will help with the cooperative agreement,0.168627,0.726364,0.605882,0.013636,37
+14,"process, it is up to the requesting community to get the agreement signed. Sister",0.169412,0.742121,0.650196,0.013939,38
+14,"Cities International will not, in any way, force a community to ""share"" and sign",0.16902,0.758182,0.623922,0.014242,39
+14,the cooperative agreement.,0.168627,0.774848,0.219216,0.013333,40
+14,"To place a relationship into Emeritus status, the mayor or highest elected official of the U.S.",0.117255,0.798485,0.736471,0.013939,41
+14,community must write a letter to the mayor of the foreign city indicating that they wish to,0.118039,0.814545,0.70902,0.013636,42
+14,"remain sister cities, but understand that the relationship will remain inactive until such time as",0.118039,0.831212,0.747451,0.013333,43
+14,both cities are able to sustain an active relationship. Sister Cities International should be,0.118039,0.847273,0.705098,0.013636,44
+14,informed in writing by the mayor of the U.S. city of the situation. Sister Cities International will,0.118039,0.863333,0.746275,0.013636,45
+15,SisterCities,0.169804,0.033333,0.238824,0.028182,1
+15,Partnership Agreement,0.516078,0.027879,0.440784,0.032424,2
+15,INTERNATIONAL,0.170196,0.06697,0.237647,0.008788,3
+15,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303,4
+15,Toolkit,0.83098,0.072727,0.127059,0.025455,5
+15,then place the partnership into Emeritus Status and will reflect this status in directories and all,0.117255,0.132424,0.751373,0.013333,6
+15,lists of sister city programs.,0.118039,0.148788,0.218431,0.013333,7
+15,"If a community wishes to terminate a sister city relationship, then a letter from the mayor or",0.118431,0.172424,0.732549,0.013333,8
+15,highest elected official of the U.S. city should be sent to the mayor of the sister city. Sister,0.118039,0.188485,0.721569,0.013636,9
+15,Cities International should be informed of this action in writing by the mayor of the U.S. city,0.118039,0.204848,0.72902,0.013333,10
+15,and Sister Cities International will then remove the partnership from its directories and all lists,0.117647,0.221212,0.746275,0.013333,11
+15,of sister city programs. We do not recommend terminating a relationship simply because it is,0.117647,0.237273,0.743529,0.013333,12
+15,"dormant. Many partnerships wax and wane over the years, and in many cases a dormant",0.117647,0.253939,0.713333,0.013333,13
+15,partnership may be reinvigorated by local members years after it has been inactive.,0.118039,0.269697,0.664314,0.013636,14
+15,General Guidelines,0.118039,0.295152,0.231765,0.016061,15
+15,In order for a sister city/county/state partnership to be recognized by Sister Cities International,0.118431,0.324242,0.754902,0.013636,16
+15,"(SCI), the two communities must sign formal documents which clearly endorse the link. This",0.118039,0.340606,0.74,0.013636,17
+15,presumes several key items: that the U.S. community is already a member of SCI and has,0.118039,0.35697,0.718039,0.013636,18
+15,followed proper procedures (e.g. passed a city council resolution declaring the intent to twin,0.117255,0.373333,0.737647,0.013636,19
+15,with the specific city); that both communities share a mutual commitment to the relationship;,0.117255,0.389394,0.740784,0.013636,20
+15,and that both have secured the necessary support structure to build a lasting relationship. You,0.117647,0.405455,0.758039,0.013333,21
+15,should check with your local sister city program to see if they have any additional requirements,0.117647,0.421818,0.760784,0.013636,22
+15,before pursuing a sister city relationship.,0.118039,0.437879,0.323137,0.013636,23
+15,"SCI often refers to these agreements as a ""Sister City Agreement"" or ""Memorandum of",0.118039,0.461515,0.696863,0.013939,24
+15,"Understanding."" However, as the following examples show, the actual name and format of",0.118039,0.477576,0.729804,0.013636,25
+15,your documents is left up to you.,0.117255,0.494242,0.262745,0.013636,26
+15,A few things to keep in mind as you draft your agreement:,0.117255,0.517879,0.463137,0.013636,27
+15,"Your agreement can range from the ceremonial, with language focusing on each city's",0.176471,0.542121,0.69098,0.013939,28
+15,"commitment to fostering understanding, cooperation, and mutual benefit to the precise,",0.176471,0.558485,0.701961,0.013333,29
+15,"with particular areas of interest, specific programs/activities, or more concrete goals",0.176078,0.574848,0.673725,0.013636,30
+15,related to anything from numbers of exchanges to economic development.,0.176863,0.591212,0.596863,0.013636,31
+15,"Don't try to include everything you plan to do. Some specifics, like particular areas of",0.177255,0.620303,0.681176,0.013939,32
+15,"interest or participating institutions are good to include. However, there's no need to",0.176471,0.636667,0.675686,0.013636,33
+15,include all the programs you plan to do if it makes the document too lengthy or limits,0.176863,0.652727,0.678824,0.013939,34
+15,the scope of projects. This is a formal document to establish the relationship; specific,0.176078,0.668788,0.684706,0.013636,35
+15,"tasks, responsibilities, or other nuts-and-bolts text related to implementation or",0.176078,0.685455,0.635686,0.013333,36
+15,administration of the partnership can be expressed more fully in a separate,0.176471,0.701212,0.600392,0.013636,37
+15,memorandum between the respective sister city committees. Your partnership,0.177255,0.717576,0.626667,0.013636,38
+15,agreement is a historical document and should not be dated or limited by being aligned,0.176471,0.733636,0.699216,0.013636,39
+15,with very specific tasks.,0.176078,0.750606,0.190196,0.013333,40
+15,Work with your counterparts. Remember that this is signed by both cities. You should,0.176078,0.779697,0.68549,0.013636,41
+15,share drafts of your agreement with your international partners and solicit feedback on,0.176471,0.795758,0.691765,0.013333,42
+15,what they'd like to see in the agreement. Be flexible to cultural or municipal priorities.,0.176471,0.811818,0.679216,0.013939,43
+15,Ask your counterparts to translate the agreement if it is drafted in English. It is,0.176078,0.841515,0.623137,0.013636,44
+15,important for the citizens of your partner community to be able to read and understand,0.176863,0.857576,0.693725,0.013939,45
+15,the commitment their city has made. Have someone in your own community who,0.176078,0.873939,0.649804,0.013636,46
+16,SisterCities,0.169804,0.033333,0.239216,0.028182,1
+16,Partnership Agreement,0.516078,0.027879,0.441176,0.032121,2
+16,INTERNATIONAL,0.170196,0.06697,0.237255,0.008788,3
+16,Connect globally. Thrive locally.,0.169804,0.08697,0.239216,0.01303,4
+16,Toolkit,0.83098,0.07303,0.126667,0.025152,5
+16,speaks that language check the foreign-language version to make sure it mirrors what,0.176471,0.132424,0.688235,0.013333,6
+16,you have in your own agreement.,0.176471,0.148788,0.264706,0.013333,7
+16,Keep it to one page. Ceremonial documents such as these partnership agreements,0.176863,0.178485,0.66549,0.013636,8
+16,work best if they can be posted in their entirety.,0.176078,0.194545,0.380392,0.013636,9
+16,Most sister city agreements include some acknowledgement of the founding principles,0.177255,0.224242,0.694902,0.013636,10
+16,"of the sister city movement- to promote peace through mutual respect, understanding,",0.176471,0.240303,0.698431,0.013333,11
+16,and cooperation.,0.176471,0.25697,0.13451,0.013333,12
+16,Consider using official letterhead and/or other embellishments such as city seals or,0.176863,0.286061,0.665882,0.013333,13
+16,logos to reflect your enhance the document. Sister city agreements are often posted at,0.176863,0.302121,0.695686,0.013636,14
+16,city hall or other municipal offices and should reflect their historical importance,0.176471,0.318485,0.630588,0.013333,15
+16,Look at other agreements your city has signed. These agreements may give you an idea,0.177255,0.347879,0.705098,0.013636,16
+16,"of what is acceptable or possible, and they may be in an easily replicable format. If you",0.176471,0.364242,0.695686,0.013636,17
+16,"cannot access older agreements please contact Sister Cities International, we may",0.176863,0.380303,0.663137,0.013636,18
+16,"have them on file, although we do not have copies of all partnership agreements.",0.176863,0.396667,0.64549,0.013636,19
+16,Documents must be signed by the top elected official of both communities.,0.177255,0.426364,0.601569,0.013333,20
+16,"Check with your mayor, city council, town clerk, et al. to make sure that the agreement",0.176863,0.455758,0.694118,0.013636,21
+16,"is OK with them. The mayor is the one putting his or her name on the paper, and you",0.176863,0.471818,0.677255,0.013333,22
+16,don't want to spend time developing an agreement which will never be signed.,0.176863,0.488182,0.629412,0.013636,23
+16,Official documents are usually signed during a formal ceremony recognizing the,0.176863,0.517576,0.638431,0.013636,24
+16,partnership. Be sure both communities receive a signed set of the official documents,0.177255,0.533939,0.683922,0.013636,25
+16,for their records.,0.176078,0.550606,0.131373,0.010606,26
+16,Remember to send your signed agreement to Sister Cities International. After we,0.177255,0.579697,0.645098,0.013636,27
+16,receive your agreement we will post the relationship in the City Directory and make sure,0.176863,0.595758,0.703137,0.013636,28
+16,it is included in our Annual Membership Directory.,0.176863,0.612121,0.398039,0.013333,29
+16,Remember that each city's sister city program is independent and can impose requirements,0.118431,0.640606,0.736471,0.013939,30
+16,"like the establishment of a committee, a review period, sustainability/funding plan, among",0.118039,0.65697,0.715686,0.013636,31
+16,"others, before sanctioning a sister city agreement. Check with your local program or mayor's",0.117647,0.672727,0.743529,0.014242,32
+16,office to see if this is the case.,0.117647,0.689091,0.241176,0.011515,33
+16,On the following pages you'll find a series of partnership agreements to give you an idea of,0.118039,0.717879,0.728627,0.013939,34
+16,"what is possible. While you should feel free to use some of the formatting and language, we",0.117255,0.734242,0.73451,0.013636,35
+16,encourage you to make your agreement your own and be creative with what you produce. If,0.117647,0.750606,0.737647,0.013636,36
+16,you are unsure about your agreement or want advice you can always solicit feedback by,0.117647,0.766667,0.708627,0.013636,37
+16,sending it to our Membership Director at akaplan@sister-cities.org or contacting us at (202),0.117647,0.782727,0.732157,0.013636,38
+16,347-8630.,0.117647,0.799394,0.080392,0.010303,39
+17,SisterCities,0.169412,0.033333,0.239608,0.028485,1
+17,Partnership Agreement,0.516471,0.027879,0.440784,0.032727,2
+17,INTERNATIONAL,0.170196,0.066667,0.238431,0.009091,3
+17,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333,4
+17,Toolkit,0.830588,0.072727,0.127843,0.025758,5
+17,"jull bubzig 2000 3,312",0.378039,0.291212,0.32549,0.019394,6
+17,ABU DHABI MUNICIPALITY & TOWN PLANNING,0.376471,0.316667,0.327451,0.016667,7
+17,AN AGREEMENT FOR THE ESTABLISHMENT OF,0.260784,0.373636,0.52549,0.012727,8
+17,SISTER CITIES RELATIONSHIP,0.337647,0.393636,0.342745,0.012121,9
+17,BETWEEN,0.454902,0.413636,0.110588,0.011212,10
+17,THE CITY OF ABU DHABI ( U. A.E),0.337255,0.432727,0.375686,0.013939,11
+17,AND,0.487843,0.452727,0.048235,0.011212,12
+17,"HOUSTON, TEXAS ( U.S.A)",0.385882,0.471515,0.298039,0.014848,13
+17,"The Sister City Program, administered by Sister Cities International, was initiated",0.221961,0.525455,0.597255,0.01303,14
+17,By the President of the United States of America in 1956 to encourage greater,0.222745,0.539394,0.561961,0.012727,15
+17,Friendship and understanding between the United States and other nations through,0.222745,0.553333,0.608235,0.012727,16
+17,Direct personal contact: and,0.222745,0.567576,0.20549,0.012424,17
+17,"In order to foster those goals, the people of Abu Dhabi and Houston, in a gesture of",0.222353,0.594242,0.603529,0.012424,18
+17,"Friendship and goodwill, agree to collaborate for the mutual benefit of their",0.222745,0.608182,0.547843,0.01303,19
+17,"Communities by exploring education, economic and cultural opportunities.",0.222353,0.622121,0.541961,0.012121,20
+17,"Abu Dhabi and Houston, sharing a common interest in energy, technology and",0.221569,0.648788,0.574118,0.012424,21
+17,"medicine, and the desire to promote mutual understanding among our citizens do",0.222353,0.66303,0.588235,0.012121,22
+17,"hereby proclaim themselves Sister Cities beginning on the 13th day of March 2001,",0.221961,0.673636,0.594118,0.015758,23
+17,the date of Houston City Council resolution estatblishing the Sister City,0.221961,0.690303,0.519608,0.01303,24
+17,relationship became effective.,0.221569,0.705152,0.217647,0.012424,25
+17,"Signed on this 26 of October 2002, in duplicate in the Arabic and English",0.221569,0.732121,0.533333,0.01303,26
+17,"Languages, both text being equally authentic.",0.221961,0.746667,0.328627,0.012727,27
+17,A,0.344314,0.768485,0.084706,0.030303,28
+17,Sheikh Mohammed bin Butti AI Hamed,0.245882,0.806364,0.366275,0.010909,29
+17,Lee P.Brown,0.729412,0.806364,0.118824,0.010303,30
+17,Chairman of Abu Dhabi Municipality,0.24549,0.823636,0.342353,0.012727,31
+17,Mayor of Houston,0.704706,0.823333,0.166667,0.012424,32
+17,&Town Planning,0.324314,0.841212,0.155686,0.012424,33
+18,SisterCities,0.169412,0.033333,0.239608,0.028485,1
+18,Partnership Agreement,0.516078,0.027879,0.441176,0.032424,2
+18,INTERNATIONAL,0.17098,0.066667,0.237255,0.009091,3
+18,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333,4
+18,Toolkit,0.83098,0.072727,0.127059,0.025758,5
+18,THE CITY OF NEW YORK,0.438824,0.262121,0.240784,0.009697,6
+18,OFFICE OF THE MAYOR,0.450196,0.27697,0.220392,0.009697,7
+18,"NEW YORK, N.Y. 10007",0.461176,0.29303,0.196863,0.010303,8
+18,THE NEW YORK CITY-LONDON SISTER CITY PARTNERSHIP,0.267451,0.355758,0.582745,0.011818,9
+18,Memorandum of Understanding,0.420392,0.371212,0.274902,0.013333,10
+18,The Sister City partnership between New York City and London will foster mutually,0.201176,0.402121,0.674118,0.014242,11
+18,beneficial solutions to common challenges for these two great cosmopolitan entities.,0.201176,0.417273,0.66902,0.013636,12
+18,"Consequently, the Sister City relationship between the two will be one of the most",0.201176,0.432727,0.652549,0.015152,13
+18,"important in their network of global partnerships, as it strives to:",0.201176,0.448182,0.50902,0.015455,14
+18,Encourage and publicize existing exchanges between London and New York City so,0.230588,0.480303,0.671373,0.015152,15
+18,that they can flourish to benefit a wider cross-section of the citizens of both;,0.230588,0.496061,0.602353,0.015152,16
+18,"Support and promote the development of new social, economic, academic and",0.230196,0.512424,0.618431,0.015455,17
+18,community programs to encourage both cities' citizens to share their experiences as a,0.229804,0.527879,0.678039,0.014848,18
+18,medium for learning from one another;,0.229804,0.543636,0.309412,0.013939,19
+18,Generate an improvement of the operation of the cities' various government agencies,0.229804,0.56,0.676078,0.014545,20
+18,by serving as a conduit of information;,0.22902,0.575758,0.307843,0.014848,21
+18,"Identify themes, common to both, that can generate new initiatives to further and",0.229412,0.591818,0.640784,0.015152,22
+18,"nurture the increasingly powerful financial, social and cultural relationships between",0.22902,0.607576,0.671373,0.014242,23
+18,the cities;,0.22902,0.624545,0.076471,0.012424,24
+18,Promote key mayoral priorities relevant to both London and New York City;,0.228627,0.639394,0.608627,0.015152,25
+18,Provide financial or in kind support to community-led programs that advance the,0.228627,0.656061,0.641569,0.013636,26
+18,aims of the Sister City partnership;,0.22902,0.672121,0.275294,0.013636,27
+18,"With the above purposes in mind, the Mayor of the City of New York and the Mayor of",0.198824,0.702424,0.697647,0.014848,28
+18,London solemnly confirm that these two cities are united by an official partnership by the,0.198824,0.718182,0.710196,0.014545,29
+18,protocol of this Memorandum of Understanding.,0.198431,0.733939,0.384314,0.015152,30
+18,This agreement will go into effect from the date of signatures.,0.310196,0.780606,0.488235,0.014545,31
+18,Signed in March of 2001,0.455686,0.796364,0.19451,0.013636,32
+18,Thedder Rudolph W. Giuliani,0.178824,0.795455,0.244314,0.100909,33
+18,Mayor,0.311373,0.894848,0.053333,0.012727,34
+18,Ken Mayor Livingstone,0.672157,0.877576,0.132941,0.029091,35
+18,New York City,0.287843,0.909091,0.121176,0.013333,36
+18,London,0.701961,0.909091,0.061569,0.010606,37
+19,SisterCities,0.169412,0.03303,0.24,0.028182,1
+19,Partnership Agreement,0.515686,0.027576,0.441961,0.03303,2
+19,INTERNATIONAL,0.169804,0.066667,0.238431,0.009091,3
+19,Connect globally. Thrive locally.,0.169412,0.08697,0.239608,0.013333,4
+19,Toolkit,0.83098,0.072727,0.127451,0.025758,5
+19,CHIC OF STATE,0.247451,0.190606,0.141961,0.036364,6
+19,City of Long Beach,0.388627,0.196667,0.476471,0.066364,7
+19,California,0.551373,0.257273,0.136471,0.033333,8
+19,Sister City Agreement,0.321961,0.305455,0.378431,0.035152,9
+19,between the,0.464706,0.352727,0.084314,0.009697,10
+19,City of Long Beach,0.38,0.378485,0.252549,0.01697,11
+19,"California, USA",0.4,0.397576,0.21098,0.016061,12
+19,and the,0.48,0.415152,0.053333,0.009091,13
+19,City of San Pablo de Manta,0.321569,0.428788,0.369804,0.01697,14
+19,"Ecuador, South America",0.347451,0.447879,0.317255,0.015152,15
+19,"In accordance with the authorization and approval expressed by the City of Long Beach,",0.261569,0.482121,0.536863,0.012121,16
+19,"California, USA, and the City of San Pablo de Manta, Ecundor, South America, it is declared",0.217647,0.492727,0.581176,0.01303,17
+19,"that a ""Sister City Agreement between the two cities is hereby established for the following",0.217647,0.502727,0.581569,0.012121,18
+19,purposes:,0.216863,0.516061,0.058039,0.009394,19
+19,(1) to promote and expand the effective and mutually beneficial cooperation between,0.278824,0.532727,0.520392,0.012424,20
+19,the people of Long Beach and the people of San Pablo de Manta; and,0.218039,0.543636,0.40549,0.012424,21
+19,"(2) to promote international goodwill, understanding, and expanded business",0.279216,0.56303,0.520784,0.012424,22
+19,"relations between the two cities and their respective nations by the exchange of people, ideas, and",0.218039,0.573636,0.581569,0.012121,23
+19,"information in a unide variety of economic, social, cultural, municipal, environmental,",0.218039,0.584242,0.581176,0.012121,24
+19,"professional, technical, youth, and other endeavors; and",0.217647,0.594848,0.333333,0.012121,25
+19,"(3) to foster and encourage charitable, scientific, trade and commerce, literary and",0.279608,0.613939,0.520784,0.012727,26
+19,educational activities between the two cities;,0.218039,0.625455,0.265882,0.009697,27
+19,This Sister City Agreement shall be officially established and shall become effective when,0.263137,0.644545,0.536863,0.012727,28
+19,"this document has been duly executed by the Mayor of Long Beach, California, USA, and the",0.218824,0.654848,0.581961,0.012424,29
+19,"Mayor of San Pablo de Manta, Ecundor, South America.",0.218431,0.665758,0.338824,0.012121,30
+19,STATE OFFICE,0.276471,0.713636,0.050588,0.048788,31
+19,Beverly 0 Neill,0.587451,0.736667,0.121961,0.013636,32
+19,"Mayor, City of Long Beach",0.542353,0.751212,0.21098,0.013636,33
+19,"California, USA",0.582745,0.765758,0.125098,0.01303,34
+19,10.2aulus,0.490588,0.771818,0.220392,0.062424,35
+19,Ing. Jorge O. Zambrano Cedeño,0.527059,0.825152,0.242745,0.013333,36
+19,"Mayor, City of San Pablo de Manta",0.505098,0.839394,0.277647,0.013636,37
+19,"Ecuador, South America",0.551765,0.854242,0.188235,0.011818,38
+19,"Dated: September 19, 2000",0.544706,0.883333,0.202745,0.01303,39
+20,SisterCities,0.169412,0.03303,0.24,0.028485,1
+20,Partnership Agreement,0.516078,0.027879,0.441176,0.032424,2
+20,INTERNATIONAL,0.170196,0.066667,0.237647,0.009091,3
+20,Connect globally. Thrive locally.,0.169412,0.08697,0.239216,0.013333,4
+20,Toolkit,0.83098,0.072727,0.127451,0.025758,5
+20,REAFFIRMATION OF SISTER CITIES DECLARATION,0.324706,0.165152,0.483529,0.013939,6
+20,adopted by,0.2,0.213333,0.080392,0.013636,7
+20,THE HONORABLE RICHARD M. DALEY,0.396078,0.214242,0.335686,0.012424,8
+20,MAYOR OF CHICAGO,0.472549,0.231212,0.18549,0.011515,9
+20,and,0.199608,0.260909,0.026275,0.010606,10
+20,THE HONORABLE ZHANG RONGMAO,0.401961,0.261212,0.323137,0.011212,11
+20,MAYOR OF SHENYANG,0.463529,0.273636,0.202353,0.011212,12
+20,ON,0.551765,0.298182,0.026667,0.011515,13
+20,"JUNE 5, 1995",0.500392,0.323636,0.128235,0.014848,14
+20,"On this the tenth anniversary of the signing of a sister city agreement, in order to further",0.255686,0.36303,0.67098,0.015152,15
+20,the traditional links of friendship between Chicago and Shenyang and to reaffirm their mutual,0.198824,0.378788,0.727843,0.015455,16
+20,"aspiration to work in unison for the benefit of their cities and nations, the Honorable Mayor",0.199608,0.394848,0.727843,0.014848,17
+20,"Richard M. Daley, Mayor of the City of Chicago, and the Honorable Zhang Rongmao, Mayor",0.199216,0.411212,0.727451,0.014242,18
+20,"of the City of Shenyang, on this fifth day of June 1995, do hereby acknowledge and reaffirm the",0.199216,0.42697,0.72549,0.014848,19
+20,sister cities agreement between the City of Chicago and the City of Shenyang.,0.199608,0.443636,0.57451,0.014242,20
+20,"The City of Chicago and the City of Shenyang on the basis of friendly cooperation,",0.256078,0.473939,0.665098,0.015152,21
+20,equality and mutual benefit will continue to develop a sister cities relationship to promote and,0.2,0.490303,0.724706,0.014242,22
+20,broaden economic cooperation and cultural exchanges between the two cities.,0.199216,0.506061,0.57451,0.014242,23
+20,The two cities do hereby declare their interest in exploring the establishment of business,0.255294,0.537273,0.668235,0.015455,24
+20,and trade relations between Chicago and Shenyang.,0.198824,0.554545,0.387843,0.013636,25
+20,"In addition, exchanges will be promoted in the area of the arts such as exhibits, music,",0.254118,0.583939,0.666667,0.015455,26
+20,dance and other cultural activities.,0.198431,0.601212,0.256471,0.010606,27
+20,"In addition, exchanges will be promoted in education and the establishment of contacts",0.254118,0.630303,0.668627,0.015758,28
+20,within educational institutions encouraged.,0.198824,0.647273,0.32,0.014242,29
+20,"In addition, we declare our intention to promote exchanges in such fields as science and",0.253725,0.678182,0.668627,0.014848,30
+20,"technology, sports, health, youth and any areas that will contribute to the prosperity and the",0.198039,0.693636,0.722745,0.015152,31
+20,further development of friendship between the people of our two cities.,0.194902,0.711515,0.525098,0.013636,32
+20,3h.5.,0.593725,0.750606,0.218039,0.06303,33
+20,THE HONORABLE RICHARD M. DALEY,0.197255,0.821515,0.303529,0.010606,34
+20,THE HONORABLE ZHANG RONGMAO,0.588627,0.819394,0.287843,0.011818,35
+20,MAYOR OF CHICAGO,0.195686,0.835758,0.164706,0.010606,36
+20,MAYOR OF SHENYANG,0.587451,0.835455,0.177647,0.010303,37
+21,Skills_based_CV.qxd 5/8/11 3:55 pm Page,0.17777,0.135381,0.308796,0.008545,1
+21,agcas,0.726169,0.191722,0.053368,0.011749,2
+21,Example of a skills-based CV,0.3894,0.205874,0.224144,0.011482,3
+21,ASHLEY GILL,0.459698,0.246195,0.082812,0.008278,4
+21,3 Lappage Court,0.2212,0.259012,0.080972,0.008545,5
+21,Telephone: 01882 652349,0.592565,0.259012,0.129555,0.008278,6
+21,"Tyler Green, Bucks.",0.220464,0.269159,0.092381,0.008278,7
+21,Mobile: 07717 121824,0.593669,0.269159,0.112992,0.006676,8
+21,HP8 4JD,0.2212,0.279306,0.040486,0.006409,9
+21,Email: ashleygill2023@gotmail.com,0.594038,0.279039,0.178874,0.008545,10
+21,Personal Details,0.221568,0.299332,0.095326,0.007744,11
+21,Summary,0.220832,0.321495,0.048215,0.008278,12
+21,Business studies with Spanish undergraduate.,0.273463,0.340988,0.229297,0.008812,13
+21,Ability to speak French and Spanish.,0.272727,0.351135,0.179242,0.008545,14
+21,Extensive business experience including an internship with Top Choice Holidays.,0.273095,0.361015,0.398233,0.008812,15
+21,Education And Qualifications,0.2212,0.381041,0.144277,0.008278,16
+21,2008 present,0.220832,0.401602,0.074715,0.008011,17
+21,Buckinghamshire Edge University,0.386824,0.401068,0.167096,0.008545,18
+21,BA International Business Studies with Spanish (expected 2:1),0.386824,0.410681,0.308796,0.008812,19
+21,Relate your degree to,0.230033,0.420027,0.100847,0.008278,20
+21,Study semester at The University of Valloid (Spain).,0.399338,0.420828,0.252852,0.008812,21
+21,the job by listing your,0.229665,0.429105,0.101583,0.008278,22
+21,Six-month work placement in Madrid.,0.399338,0.431242,0.188811,0.008545,23
+21,relevant modules/,0.230033,0.438718,0.085388,0.007744,24
+21,Relevant modules included: Business Planning; Sales Promotion and,0.399338,0.441389,0.338241,0.008545,25
+21,dissertation.,0.230033,0.448064,0.057784,0.006676,26
+21,Marketing; and Business Operations Management.,0.398969,0.451268,0.25322,0.008812,27
+21,2000 2007,0.2212,0.467824,0.061833,0.006409,28
+21,Freebridge School,0.386824,0.46729,0.087965,0.008545,29
+21,"A-Levels: Business Studies (B), French (C)",0.386088,0.476903,0.200221,0.008812,30
+21,"8 GCSEs including Maths, English, Spanish and French",0.386824,0.487583,0.266838,0.008545,31
+21,Work History,0.220832,0.509212,0.065513,0.008278,32
+21,2008 2011,0.220832,0.529506,0.061833,0.006409,33
+21,Buckinghamshire Edge University Librarian/tour guide,0.386824,0.528972,0.277144,0.008812,34
+21,General administrative and customer service roles.,0.399338,0.539119,0.25138,0.006676,35
+21,Briefly list,0.707766,0.536716,0.045639,0.008011,36
+21,your relevant,0.70703,0.546061,0.061465,0.008011,37
+21,2011 (Feb-Aug),0.2212,0.55514,0.078027,0.008812,38
+21,Audigest S.A. (Madrid) - Audit Assistant,0.386456,0.554873,0.199485,0.009079,39
+21,duties.,0.707398,0.555674,0.030916,0.006409,40
+21,Six months' work experience in an international bank.,0.399338,0.565287,0.267575,0.008545,41
+21,Liaising with colleagues and clients in English and Spanish.,0.399338,0.575434,0.292602,0.008545,42
+21,2010 (June-Dec),0.220832,0.591188,0.082444,0.008278,43
+21,Finsbury's supermarket (Hazelbridge) — Supervisor,0.386824,0.591188,0.250644,0.008812,44
+21,Managing a small team.,0.398969,0.601602,0.121089,0.008545,45
+21,Customer service in a busy competitive environment.,0.398969,0.611215,0.264262,0.008545,46
+21,2010 (Jan-Aug),0.2212,0.627236,0.077291,0.008812,47
+21,Top Choice Holidays and Flights Ltd (Low Wycombe),0.386088,0.627503,0.257637,0.008812,48
+21,Financial Assistant/Supervisor,0.386824,0.637383,0.15127,0.008812,49
+21,Working in a range of teams to manage complex financial processes.,0.398969,0.64753,0.341921,0.008812,50
+21,2007 (Jul-Aug),0.220832,0.663284,0.074347,0.008812,51
+21,Dogs Protection League - General Assistant,0.386824,0.663818,0.216783,0.008812,52
+21,Dealing with enquiries and selling packages to a range of clients.,0.399706,0.673431,0.321678,0.009079,53
+21,2006 (Jan-Dec),0.220832,0.689453,0.076187,0.009079,54
+21,McHenry's Restaurant (Low Wycombe) - Supervisor,0.386456,0.68972,0.256533,0.009079,55
+21,Voluntary Experience,0.220464,0.708411,0.106367,0.008545,56
+21,2007/2011,0.220832,0.728438,0.055208,0.008011,57
+21,Teaching English in Mexico/Spain,0.386088,0.727904,0.167832,0.009079,58
+21,Interests,0.2212,0.748465,0.043062,0.006676,59
+21,Active member of University Business Club — Winner of the 'Bucks Best Business Pitch' award in 2010 Enterprise,0.220464,0.768224,0.556864,0.009079,60
+21,"week, judged by Michael Eavis.",0.220464,0.778104,0.15311,0.008812,61
+22,Skills_based_CV.qxd 5/8/11 3:55 pm Page,0.17777,0.135381,0.308428,0.008545,1
+22,Make sure you carefully assess,0.468531,0.23498,0.142068,0.008011,2
+22,Skills And Achievements,0.220832,0.245394,0.121457,0.006676,3
+22,the job advert/job description,0.468163,0.244326,0.139124,0.008278,4
+22,and address all the skills they,0.468531,0.253672,0.13618,0.008278,5
+22,Effective communication,0.2212,0.265421,0.123298,0.006676,6
+22,require.,0.468531,0.263017,0.034965,0.008011,7
+22,"Able to communicate effectively with a wide range of clients and colleagues, by showing interest, carefully",0.233714,0.275567,0.530364,0.008545,8
+22,"listening to needs and appropriately adjusting my message, as demonstrated during my time at Finsbury's",0.23445,0.285447,0.528892,0.008812,9
+22,Supermarket.,0.234082,0.295861,0.066618,0.008278,10
+22,Strong presentation skills and confidence demonstrated by experience of delivering presentations in different,0.23445,0.305474,0.543614,0.008812,11
+22,languages to groups of five to fifty.,0.234082,0.315621,0.172617,0.008812,12
+22,Customer service,0.220832,0.335915,0.085388,0.006676,13
+22,Ability to quickly build rapport with customers and calmly deal with any problems as shown during my retail,0.233714,0.345527,0.541038,0.008812,14
+22,experience in high pressure environments.,0.234082,0.355941,0.210526,0.008278,15
+22,"Capacity to maintain professional relationships through email and other written correspondence, for example,",0.234082,0.365554,0.548767,0.008812,16
+22,"at Audigest in Madrid, where I built longstanding business relationships with customers and colleagues across",0.233714,0.375701,0.549871,0.008812,17
+22,the globe.,0.233714,0.385848,0.049687,0.008278,18
+22,Teamwork,0.220464,0.406142,0.052632,0.006409,19
+22,"At Top Choice Holidays demonstrated excellent teamwork skills in a busy financial environment, such as an",0.233346,0.415754,0.532573,0.008812,20
+22,"ability to listen to clients and managers, perform my role to a high level and support colleagues, resulting in",0.234082,0.425634,0.535885,0.008812,21
+22,promotion.,0.234082,0.436048,0.05484,0.008545,22
+22,Administration,0.220464,0.456075,0.075083,0.006409,23
+22,Prove you have each of the,0.639676,0.453672,0.123666,0.008278,24
+22,"Excellent ability to plan ahead and manage time effectively, for example,",0.23445,0.465688,0.360692,0.008812,25
+22,skills required by outlining,0.63894,0.463017,0.12293,0.008278,26
+22,managing complex roles during my internship at Top Choice Holidays.,0.23445,0.476101,0.346338,0.008545,27
+22,where you performed them,0.63894,0.472363,0.128082,0.008278,28
+22,Gathered data from a wide range of sources during my dissertation,0.234082,0.485714,0.334928,0.008812,29
+22,and how you performed,0.639308,0.481709,0.111888,0.008278,30
+22,them well.,0.63894,0.491055,0.048951,0.006409,31
+22,"whilst balancing my other studies and two jobs, resulting in a 73% grade.",0.233346,0.495861,0.365109,0.008812,32
+22,Experience of travellers' needs,0.2212,0.515888,0.150534,0.008545,33
+22,Recent travel consultancy experience gives me an in-depth understanding of the expectations of holiday,0.23445,0.525768,0.518955,0.008812,34
+22,customers and the competitive nature of the industry.,0.234082,0.535915,0.269047,0.008812,35
+22,International travel experience and language ability give me an empathy with travellers and a passion for,0.234082,0.545794,0.524107,0.008812,36
+22,helping them find a unique holiday experience.,0.234082,0.555941,0.23445,0.008812,37
+22,Initiative,0.2212,0.576235,0.044166,0.006676,38
+22,Self-funding an evening course in bookkeeping during my first accountancy role demonstrated my ability to,0.234082,0.585848,0.535149,0.008812,39
+22,plan ahead and take control of my career.,0.23445,0.595995,0.205006,0.008545,40
+22,Successful study and work in Spain and Mexico show that I can creatively develop my skills and experience and,0.234082,0.605874,0.551711,0.008545,41
+22,adapt to new and different environments.,0.234082,0.616288,0.208686,0.008278,42
+22,Sales knowledge,0.220464,0.636315,0.083916,0.008011,43
+22,Wide experience of financial roles gives me an awareness of the tight monetary pressures which drive UK,0.234082,0.645928,0.525212,0.009346,44
+22,service industries.,0.234082,0.656609,0.088333,0.006943,45
+22,Raised sales at The Dogs Protection League by 12% by up selling add-on packages to new and existing,0.23445,0.665955,0.505705,0.009079,46
+22,customers.,0.234082,0.67717,0.054472,0.006142,47
+22,Language ability,0.2212,0.696395,0.082444,0.008812,48
+22,"Spanish fluency obtained working overseas, French semi-fluent.",0.233714,0.706008,0.323151,0.009079,49
+22,Referees,0.2212,0.726569,0.041958,0.006676,50
+22,Include all your referee details including their email and,0.351859,0.722029,0.259109,0.008545,51
+22,phone number (but ask for their permission first).,0.352227,0.731108,0.230401,0.008545,52
+22,"Professional: Mr. Jose Andreas, Management Accountant, Audigest, Avenida de Concha Espina 2, Madrid, ES-",0.2212,0.746328,0.537725,0.008812,53
+22,"28036, +34 91 398 5476, j.andreas@audigest.es",0.2212,0.756475,0.238498,0.008278,54
+22,"Academic: Dr. Jane Luffle, Personal Tutor, Buckinghamshire Edge University, Due Road, Low Wycombe, Bucks,",0.220464,0.776502,0.536621,0.008812,55
+22,"HD15 3DL, 01628 435 6784, j.luffle@bedge.ac.uk",0.2212,0.786382,0.244755,0.008545,56