Spaces:
Paused
initial release v0.1.0: headless browser rendering API
Browse filesVeilRender is a self-hostable headless browser rendering API built with
zerodep modules and CloakBrowser (stealth Chromium). Accepts a URL,
renders it with the browser, and returns content as HTML, Markdown, or
readability-extracted text.
Features:
- POST /render — URL to rendered HTML/Markdown/readability
- POST /screenshot — URL to PNG screenshot
- GET /health — liveness check
- CDP WebSocket proxy on /cdp for direct browser control
- Token-based authentication via Bearer header or query param
- Per-request browser context isolation with concurrency control
- CloakBrowser with 58 source-level anti-fingerprint patches
- 13 vendored zerodep modules, only external dep is cloakbrowser
- Docker image based on python:3.12-slim
- GitHub CI: pre-commit (ruff + ty), multi-arch Docker (amd64/arm64)
- Auto-deploy to HF Spaces and Docker Hub/GHCR
- Bilingual README (en/zh)
- .github/workflows/ci.yml +125 -0
- .gitignore +28 -0
- .pre-commit-config.yaml +20 -0
- CHANGELOG.md +58 -0
- CLAUDE.md +51 -0
- Dockerfile +43 -0
- Makefile +113 -0
- README.md +0 -1
- README.md +1 -0
- README_en.md +88 -0
- README_zh.md +88 -0
- pyproject.toml +61 -0
- src/veilrender/__init__.py +3 -0
- src/veilrender/__main__.py +6 -0
- src/veilrender/_vendor/__init__.py +2 -0
- src/veilrender/_vendor/benchmark_compare.py +323 -0
- src/veilrender/_vendor/cache.py +1023 -0
- src/veilrender/_vendor/config.py +713 -0
- src/veilrender/_vendor/dotenv.py +514 -0
- src/veilrender/_vendor/httpserver.py +1007 -0
- src/veilrender/_vendor/jsonc.py +352 -0
- src/veilrender/_vendor/markdown.py +904 -0
- src/veilrender/_vendor/readability.py +1002 -0
- src/veilrender/_vendor/retry.py +503 -0
- src/veilrender/_vendor/soup.py +998 -0
- src/veilrender/_vendor/structlog.py +888 -0
- src/veilrender/_vendor/useragent.py +475 -0
- src/veilrender/_vendor/yaml.py +1124 -0
- src/veilrender/app.py +158 -0
- src/veilrender/auth.py +39 -0
- src/veilrender/browser.py +172 -0
- src/veilrender/cdp_proxy.py +314 -0
- src/veilrender/config.py +25 -0
- src/veilrender/models.py +109 -0
- src/veilrender/routes/__init__.py +1 -0
- src/veilrender/routes/health.py +17 -0
- src/veilrender/routes/render.py +122 -0
- src/veilrender/routes/screenshot.py +65 -0
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
branches: [master]
|
| 6 |
+
tags: ["v*"]
|
| 7 |
+
pull_request:
|
| 8 |
+
branches: [master]
|
| 9 |
+
|
| 10 |
+
jobs:
|
| 11 |
+
lint:
|
| 12 |
+
runs-on: ubuntu-latest
|
| 13 |
+
steps:
|
| 14 |
+
- uses: actions/checkout@v6
|
| 15 |
+
|
| 16 |
+
- name: Set up Python 3.10
|
| 17 |
+
uses: actions/setup-python@v6
|
| 18 |
+
with:
|
| 19 |
+
python-version: "3.10"
|
| 20 |
+
|
| 21 |
+
- name: Install dependencies
|
| 22 |
+
run: |
|
| 23 |
+
python -m pip install --upgrade pip
|
| 24 |
+
pip install -e ".[dev]"
|
| 25 |
+
|
| 26 |
+
- name: Run pre-commit checks
|
| 27 |
+
uses: pre-commit/action@v3.0.1
|
| 28 |
+
|
| 29 |
+
docker:
|
| 30 |
+
needs: lint
|
| 31 |
+
if: github.event_name == 'push'
|
| 32 |
+
runs-on: ubuntu-latest
|
| 33 |
+
permissions:
|
| 34 |
+
contents: read
|
| 35 |
+
packages: write
|
| 36 |
+
steps:
|
| 37 |
+
- uses: actions/checkout@v6
|
| 38 |
+
|
| 39 |
+
- name: Set up QEMU
|
| 40 |
+
uses: docker/setup-qemu-action@v3
|
| 41 |
+
|
| 42 |
+
- name: Set up Docker Buildx
|
| 43 |
+
uses: docker/setup-buildx-action@v3
|
| 44 |
+
|
| 45 |
+
- name: Login to GHCR
|
| 46 |
+
uses: docker/login-action@v3
|
| 47 |
+
with:
|
| 48 |
+
registry: ghcr.io
|
| 49 |
+
username: ${{ github.actor }}
|
| 50 |
+
password: ${{ secrets.GITHUB_TOKEN }}
|
| 51 |
+
|
| 52 |
+
- name: Login to Docker Hub
|
| 53 |
+
uses: docker/login-action@v3
|
| 54 |
+
with:
|
| 55 |
+
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
| 56 |
+
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
| 57 |
+
|
| 58 |
+
- name: Docker metadata
|
| 59 |
+
id: meta
|
| 60 |
+
uses: docker/metadata-action@v5
|
| 61 |
+
with:
|
| 62 |
+
images: |
|
| 63 |
+
ghcr.io/${{ github.repository }}
|
| 64 |
+
docker.io/${{ github.repository }}
|
| 65 |
+
tags: |
|
| 66 |
+
type=ref,event=branch
|
| 67 |
+
type=semver,pattern={{version}}
|
| 68 |
+
type=semver,pattern={{major}}.{{minor}}
|
| 69 |
+
type=sha,prefix=
|
| 70 |
+
|
| 71 |
+
- name: Build and push
|
| 72 |
+
uses: docker/build-push-action@v6
|
| 73 |
+
with:
|
| 74 |
+
context: .
|
| 75 |
+
platforms: linux/amd64,linux/arm64
|
| 76 |
+
push: true
|
| 77 |
+
tags: ${{ steps.meta.outputs.tags }}
|
| 78 |
+
labels: ${{ steps.meta.outputs.labels }}
|
| 79 |
+
cache-from: type=gha
|
| 80 |
+
cache-to: type=gha,mode=max
|
| 81 |
+
|
| 82 |
+
deploy-hf:
|
| 83 |
+
needs: lint
|
| 84 |
+
if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v'))
|
| 85 |
+
runs-on: ubuntu-latest
|
| 86 |
+
steps:
|
| 87 |
+
- uses: actions/checkout@v6
|
| 88 |
+
with:
|
| 89 |
+
fetch-depth: 0
|
| 90 |
+
lfs: true
|
| 91 |
+
|
| 92 |
+
- name: Prepare HF Space files
|
| 93 |
+
run: |
|
| 94 |
+
cat > README.md <<'FRONTMATTER'
|
| 95 |
+
---
|
| 96 |
+
title: VeilRender
|
| 97 |
+
emoji: 👻
|
| 98 |
+
colorFrom: gray
|
| 99 |
+
colorTo: purple
|
| 100 |
+
sdk: docker
|
| 101 |
+
app_port: 7860
|
| 102 |
+
pinned: false
|
| 103 |
+
---
|
| 104 |
+
|
| 105 |
+
# VeilRender
|
| 106 |
+
|
| 107 |
+
Headless browser rendering API. See [GitHub repo](https://github.com/${{ github.repository }}) for full documentation.
|
| 108 |
+
FRONTMATTER
|
| 109 |
+
sed -i 's/^ //' README.md
|
| 110 |
+
rm -rf .github .pre-commit-config.yaml CLAUDE.md README_en.md README_zh.md Makefile
|
| 111 |
+
|
| 112 |
+
- name: Push to HF Space
|
| 113 |
+
env:
|
| 114 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
| 115 |
+
HF_SPACE: ${{ vars.HF_SPACE }} # e.g. oaklight/veilrender
|
| 116 |
+
run: |
|
| 117 |
+
if [ -z "$HF_SPACE" ]; then
|
| 118 |
+
echo "::warning::HF_SPACE variable not set, skipping HF deploy"
|
| 119 |
+
exit 0
|
| 120 |
+
fi
|
| 121 |
+
git config user.name "github-actions[bot]"
|
| 122 |
+
git config user.email "github-actions[bot]@users.noreply.github.com"
|
| 123 |
+
git add -A
|
| 124 |
+
git commit -m "deploy from GitHub $(git rev-parse --short HEAD@{1})" --allow-empty
|
| 125 |
+
git push --force "https://oauth2:${HF_TOKEN}@huggingface.co/spaces/${HF_SPACE}" HEAD:main
|
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Python
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*.egg-info/
|
| 5 |
+
dist/
|
| 6 |
+
build/
|
| 7 |
+
*.egg
|
| 8 |
+
|
| 9 |
+
# Virtual environments
|
| 10 |
+
.venv/
|
| 11 |
+
venv/
|
| 12 |
+
|
| 13 |
+
# IDE
|
| 14 |
+
.vscode/
|
| 15 |
+
.idea/
|
| 16 |
+
|
| 17 |
+
# OS
|
| 18 |
+
.DS_Store
|
| 19 |
+
|
| 20 |
+
# Docker
|
| 21 |
+
.docker/
|
| 22 |
+
|
| 23 |
+
# Playwright
|
| 24 |
+
.playwright-mcp/
|
| 25 |
+
|
| 26 |
+
# Claude
|
| 27 |
+
.claude/
|
| 28 |
+
.ruff_cache/
|
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
repos:
|
| 2 |
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
| 3 |
+
rev: v0.15.6
|
| 4 |
+
hooks:
|
| 5 |
+
- id: ruff
|
| 6 |
+
args: [--fix]
|
| 7 |
+
exclude: ^src/veilrender/_vendor/
|
| 8 |
+
- id: ruff-format
|
| 9 |
+
exclude: ^src/veilrender/_vendor/
|
| 10 |
+
|
| 11 |
+
- repo: local
|
| 12 |
+
hooks:
|
| 13 |
+
- id: ty
|
| 14 |
+
name: ty check
|
| 15 |
+
description: Type check src/ with ty.
|
| 16 |
+
entry: ty check src/
|
| 17 |
+
language: system
|
| 18 |
+
pass_filenames: false
|
| 19 |
+
require_serial: true
|
| 20 |
+
types: [python]
|
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Changelog
|
| 2 |
+
|
| 3 |
+
All notable changes to this project will be documented in this file.
|
| 4 |
+
|
| 5 |
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
| 6 |
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
| 7 |
+
|
| 8 |
+
## [Unreleased]
|
| 9 |
+
|
| 10 |
+
## [0.1.0] - 2026-06-12
|
| 11 |
+
|
| 12 |
+
### Added
|
| 13 |
+
|
| 14 |
+
- Core rendering API with three endpoints:
|
| 15 |
+
- `POST /render` — URL → rendered HTML / Markdown / readability-extracted text
|
| 16 |
+
- `POST /screenshot` — URL → PNG screenshot
|
| 17 |
+
- `GET /health` — liveness check
|
| 18 |
+
- CDP WebSocket proxy at `/cdp` for direct browser control ([#3])
|
| 19 |
+
- Token-based authentication via `Authorization: Bearer` header or `?token=` query param
|
| 20 |
+
- Per-request browser context isolation with configurable concurrency (`VEILRENDER_MAX_CONCURRENT`)
|
| 21 |
+
- [CloakBrowser](https://github.com/CloakHQ/CloakBrowser) integration — stealth Chromium with 58 source-level anti-fingerprint patches ([#14], [#15])
|
| 22 |
+
- 13 vendored [zerodep](https://github.com/Oaklight/zerodep) modules — only external dependency is `cloakbrowser`
|
| 23 |
+
- Docker image based on `python:3.12-slim` with multi-arch support (amd64/arm64)
|
| 24 |
+
- GitHub CI pipeline: pre-commit (ruff + ty), Docker build to GHCR + Docker Hub, auto-deploy to HF Spaces
|
| 25 |
+
- PyPI publishing on tag push
|
| 26 |
+
- `Makefile` with `deploy-dev` (VPS) and `deploy-hf` (HF Spaces) targets
|
| 27 |
+
- Bilingual README (en/zh)
|
| 28 |
+
|
| 29 |
+
### Fixed
|
| 30 |
+
|
| 31 |
+
- CDP proxy: preserve FIN bit during WebSocket frame forwarding ([#4], [#12])
|
| 32 |
+
- HTTP multiplexer: pipe remaining POST body data instead of `feed_eof()` ([#5], [#9])
|
| 33 |
+
- CDP proxy: add 16 MB max frame size limit to prevent OOM ([#6], [#10])
|
| 34 |
+
- CDP proxy: cancel both directions when one side closes ([#7], [#11])
|
| 35 |
+
- CDP proxy: use random WebSocket key per connection and forward `Sec-WebSocket-Protocol` ([#8], [#13])
|
| 36 |
+
- Docker: use `CLOAKBROWSER_CACHE_DIR` so build-time binary download is available at runtime ([#16], [#17])
|
| 37 |
+
- Launch Chromium directly via `subprocess.Popen` + `connect_over_cdp()` instead of Playwright's `launch()`, which overrides `--remote-debugging-port`
|
| 38 |
+
|
| 39 |
+
[Unreleased]: https://github.com/Oaklight/veilrender/compare/v0.1.0...HEAD
|
| 40 |
+
[0.1.0]: https://github.com/Oaklight/veilrender/releases/tag/v0.1.0
|
| 41 |
+
|
| 42 |
+
[#1]: https://github.com/Oaklight/veilrender/issues/1
|
| 43 |
+
[#2]: https://github.com/Oaklight/veilrender/issues/2
|
| 44 |
+
[#3]: https://github.com/Oaklight/veilrender/issues/3
|
| 45 |
+
[#4]: https://github.com/Oaklight/veilrender/issues/4
|
| 46 |
+
[#5]: https://github.com/Oaklight/veilrender/issues/5
|
| 47 |
+
[#6]: https://github.com/Oaklight/veilrender/issues/6
|
| 48 |
+
[#7]: https://github.com/Oaklight/veilrender/issues/7
|
| 49 |
+
[#8]: https://github.com/Oaklight/veilrender/issues/8
|
| 50 |
+
[#9]: https://github.com/Oaklight/veilrender/pull/9
|
| 51 |
+
[#10]: https://github.com/Oaklight/veilrender/pull/10
|
| 52 |
+
[#11]: https://github.com/Oaklight/veilrender/pull/11
|
| 53 |
+
[#12]: https://github.com/Oaklight/veilrender/pull/12
|
| 54 |
+
[#13]: https://github.com/Oaklight/veilrender/pull/13
|
| 55 |
+
[#14]: https://github.com/Oaklight/veilrender/issues/14
|
| 56 |
+
[#15]: https://github.com/Oaklight/veilrender/pull/15
|
| 57 |
+
[#16]: https://github.com/Oaklight/veilrender/issues/16
|
| 58 |
+
[#17]: https://github.com/Oaklight/veilrender/pull/17
|
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CLAUDE.md — VeilRender
|
| 2 |
+
|
| 3 |
+
## What this project is
|
| 4 |
+
|
| 5 |
+
Headless browser rendering API. Accepts a URL, renders it with Playwright +
|
| 6 |
+
Chromium, returns HTML/Markdown/readability content. Primary deployment target:
|
| 7 |
+
HF Spaces (free tier). Also runs as standalone Docker container.
|
| 8 |
+
|
| 9 |
+
## Architecture
|
| 10 |
+
|
| 11 |
+
- **Zero external deps** except `playwright` — uses vendored `zerodep` modules
|
| 12 |
+
in `src/veilrender/_vendor/` for HTTP server, HTML parsing, readability, etc.
|
| 13 |
+
- **`httpserver`** (zerodep) as the async HTTP framework — not FastAPI
|
| 14 |
+
- **Playwright** for browser control — shared browser instance, per-request
|
| 15 |
+
`BrowserContext` isolation
|
| 16 |
+
- **Stateless** — no session management, no database
|
| 17 |
+
|
| 18 |
+
## Repository layout
|
| 19 |
+
|
| 20 |
+
```
|
| 21 |
+
src/veilrender/
|
| 22 |
+
├── app.py # HTTP server setup, route registration, main()
|
| 23 |
+
├── config.py # Settings from env vars (VEILRENDER_ prefix)
|
| 24 |
+
├── auth.py # Token verification
|
| 25 |
+
├── browser.py # Playwright browser lifecycle
|
| 26 |
+
├── models.py # Request/response dataclasses
|
| 27 |
+
├── routes/ # Route handlers (render, screenshot, health)
|
| 28 |
+
└── _vendor/ # zerodep modules — DO NOT EDIT manually
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
## Commands
|
| 32 |
+
|
| 33 |
+
```bash
|
| 34 |
+
make dev # Run dev server on :7860
|
| 35 |
+
make build # Docker build
|
| 36 |
+
make run # Docker run
|
| 37 |
+
make lint # ruff check --fix && ruff format
|
| 38 |
+
make typecheck # ty check
|
| 39 |
+
make vendor # Re-vendor zerodep modules
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
## Files to never edit
|
| 43 |
+
|
| 44 |
+
- `src/veilrender/_vendor/**` — vendored zerodep modules, update via
|
| 45 |
+
`zerodep update` from ~/projects/zerodep
|
| 46 |
+
|
| 47 |
+
## Definition of done
|
| 48 |
+
|
| 49 |
+
1. `ruff check --fix && ruff format` on changed Python files
|
| 50 |
+
2. Server starts and `/health` returns 200
|
| 51 |
+
3. `/render` returns valid content for a test URL
|
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.12-slim
|
| 2 |
+
|
| 3 |
+
# System libraries required by headless Chromium
|
| 4 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 5 |
+
libatk1.0-0 \
|
| 6 |
+
libatk-bridge2.0-0 \
|
| 7 |
+
libcups2 \
|
| 8 |
+
libxcb1 \
|
| 9 |
+
libxkbcommon0 \
|
| 10 |
+
libatspi2.0-0 \
|
| 11 |
+
libx11-6 \
|
| 12 |
+
libxcomposite1 \
|
| 13 |
+
libxdamage1 \
|
| 14 |
+
libxext6 \
|
| 15 |
+
libxfixes3 \
|
| 16 |
+
libxrandr2 \
|
| 17 |
+
libgbm1 \
|
| 18 |
+
libcairo2 \
|
| 19 |
+
libpango-1.0-0 \
|
| 20 |
+
libnss3 \
|
| 21 |
+
libnspr4 \
|
| 22 |
+
libasound2t64 \
|
| 23 |
+
libdbus-1-3 \
|
| 24 |
+
fonts-liberation \
|
| 25 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 26 |
+
|
| 27 |
+
# HF Spaces runs as non-root user with UID 1000
|
| 28 |
+
RUN useradd -m -u 1000 user || true
|
| 29 |
+
|
| 30 |
+
WORKDIR /app
|
| 31 |
+
|
| 32 |
+
# Install Python package (includes cloakbrowser + playwright)
|
| 33 |
+
COPY pyproject.toml .
|
| 34 |
+
COPY src/ src/
|
| 35 |
+
RUN pip install --no-cache-dir .
|
| 36 |
+
|
| 37 |
+
# Pre-download CloakBrowser Chromium binary so the image is self-contained
|
| 38 |
+
RUN python -c "from cloakbrowser import ensure_binary; ensure_binary()"
|
| 39 |
+
|
| 40 |
+
USER 1000
|
| 41 |
+
EXPOSE 7860
|
| 42 |
+
|
| 43 |
+
CMD ["python", "-m", "veilrender"]
|
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.PHONY: dev build run lint typecheck vendor clean deploy-dev deploy-hf help
|
| 2 |
+
|
| 3 |
+
REGISTRY_MIRROR ?= docker.io
|
| 4 |
+
DOCKER_IMAGE := oaklight/veilrender
|
| 5 |
+
VERSION := $(shell grep -oE '__version__[[:space:]]*=[[:space:]]*"[^"]+"' src/veilrender/__init__.py | grep -oE '"[^"]+"' | tr -d '"' || echo "0.1.0")
|
| 6 |
+
|
| 7 |
+
dev:
|
| 8 |
+
python -m veilrender
|
| 9 |
+
|
| 10 |
+
build:
|
| 11 |
+
docker build -t $(DOCKER_IMAGE):latest .
|
| 12 |
+
|
| 13 |
+
run:
|
| 14 |
+
docker run --rm -p 7860:7860 -e VEILRENDER_API_TOKEN=dev-token $(DOCKER_IMAGE):latest
|
| 15 |
+
|
| 16 |
+
lint:
|
| 17 |
+
ruff check --fix src/veilrender/ --exclude src/veilrender/_vendor/
|
| 18 |
+
ruff format src/veilrender/ --exclude src/veilrender/_vendor/
|
| 19 |
+
|
| 20 |
+
typecheck:
|
| 21 |
+
ty check src/veilrender/
|
| 22 |
+
|
| 23 |
+
vendor:
|
| 24 |
+
cd ~/projects/zerodep && python zerodep.py add httpserver readability soup markdown cache config useragent retry structlog -d $(CURDIR)/src/veilrender/_vendor/ -y -f
|
| 25 |
+
|
| 26 |
+
clean:
|
| 27 |
+
rm -rf build/ dist/ *.egg-info/ src/*.egg-info/
|
| 28 |
+
find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
|
| 29 |
+
|
| 30 |
+
# ──────────────────────────────────────────────
|
| 31 |
+
# Dev Deployment
|
| 32 |
+
# ──────────────────────────────────────────────
|
| 33 |
+
|
| 34 |
+
SSH_TARGET ?=
|
| 35 |
+
DEVTEST_STACK ?= /dockervol/dockge/stacks/veilrender
|
| 36 |
+
|
| 37 |
+
# Build dev-test image, push to remote VPS, restart container.
|
| 38 |
+
# Usage: make deploy-dev SSH_TARGET=cloud.usa2
|
| 39 |
+
deploy-dev:
|
| 40 |
+
ifndef SSH_TARGET
|
| 41 |
+
$(error SSH_TARGET is required. Usage: make deploy-dev SSH_TARGET=cloud.usa2)
|
| 42 |
+
endif
|
| 43 |
+
@set -e; \
|
| 44 |
+
COMMIT=$$(git rev-parse --short HEAD); \
|
| 45 |
+
DEV_VER="$(VERSION).dev0+g$$COMMIT"; \
|
| 46 |
+
echo "==> Building Docker image ($$DEV_VER)..."; \
|
| 47 |
+
docker build -t $(DOCKER_IMAGE):dev-test -q .; \
|
| 48 |
+
echo "==> Deploying to $(SSH_TARGET) via zstd..."; \
|
| 49 |
+
docker save $(DOCKER_IMAGE):dev-test | zstd -3 | ssh $(SSH_TARGET) \
|
| 50 |
+
'zstd -d | docker load && \
|
| 51 |
+
cd $(DEVTEST_STACK) && \
|
| 52 |
+
sed -i "s|image:.*|image: $(DOCKER_IMAGE):dev-test|" compose.yaml && \
|
| 53 |
+
docker compose up -d --force-recreate && \
|
| 54 |
+
sleep 5 && \
|
| 55 |
+
echo "=== Health check ===" && \
|
| 56 |
+
curl -sS -o /dev/null -w "%{http_code} /health\n" http://127.0.0.1:7860/ || true'; \
|
| 57 |
+
echo "==> VPS dev-test deployed successfully ($$DEV_VER)."
|
| 58 |
+
|
| 59 |
+
# Deploy to Hugging Face Spaces by pushing current code.
|
| 60 |
+
# Usage: make deploy-hf HF_SPACE=oaklight/veilrender
|
| 61 |
+
HF_SPACE ?=
|
| 62 |
+
deploy-hf:
|
| 63 |
+
ifndef HF_SPACE
|
| 64 |
+
$(error HF_SPACE is required. Usage: make deploy-hf HF_SPACE=oaklight/veilrender)
|
| 65 |
+
endif
|
| 66 |
+
@set -e; \
|
| 67 |
+
COMMIT=$$(git rev-parse --short HEAD); \
|
| 68 |
+
echo "==> Deploying to HF Space $(HF_SPACE) ($$COMMIT)..."; \
|
| 69 |
+
TMP=$$(mktemp -d); \
|
| 70 |
+
git clone --depth 1 . "$$TMP/repo"; \
|
| 71 |
+
cd "$$TMP/repo"; \
|
| 72 |
+
printf '%s\n' \
|
| 73 |
+
'---' \
|
| 74 |
+
'title: VeilRender' \
|
| 75 |
+
'emoji: 👻' \
|
| 76 |
+
'colorFrom: gray' \
|
| 77 |
+
'colorTo: purple' \
|
| 78 |
+
'sdk: docker' \
|
| 79 |
+
'app_port: 7860' \
|
| 80 |
+
'pinned: false' \
|
| 81 |
+
'---' \
|
| 82 |
+
'' \
|
| 83 |
+
'# VeilRender' \
|
| 84 |
+
'' \
|
| 85 |
+
"Headless browser rendering API. See [GitHub repo]($$(git remote get-url origin | sed 's/\.git$$//; s|git@github.com:|https://github.com/|')) for full documentation." \
|
| 86 |
+
> README.md; \
|
| 87 |
+
rm -rf .github .pre-commit-config.yaml CLAUDE.md README_en.md README_zh.md Makefile; \
|
| 88 |
+
git add -A; \
|
| 89 |
+
git commit -m "deploy $$COMMIT" --allow-empty; \
|
| 90 |
+
git push --force "https://oauth2:$${HF_TOKEN}@huggingface.co/spaces/$(HF_SPACE)" HEAD:main; \
|
| 91 |
+
rm -rf "$$TMP"; \
|
| 92 |
+
echo "==> HF Space deployed successfully ($$COMMIT)."
|
| 93 |
+
|
| 94 |
+
help:
|
| 95 |
+
@echo "Available targets:"
|
| 96 |
+
@echo " dev - Run development server on :7860"
|
| 97 |
+
@echo " build - Build Docker image"
|
| 98 |
+
@echo " run - Run Docker container"
|
| 99 |
+
@echo " lint - Run ruff check and format"
|
| 100 |
+
@echo " typecheck - Run ty check"
|
| 101 |
+
@echo " vendor - Re-vendor zerodep modules"
|
| 102 |
+
@echo " clean - Remove build artifacts"
|
| 103 |
+
@echo " deploy-dev - Build dev image and deploy to remote VPS"
|
| 104 |
+
@echo " deploy-hf - Deploy to Hugging Face Spaces"
|
| 105 |
+
@echo ""
|
| 106 |
+
@echo "Variables:"
|
| 107 |
+
@echo " SSH_TARGET=<host> - SSH target for deploy-dev (required)"
|
| 108 |
+
@echo " HF_SPACE=<user/repo> - HF Space for deploy-hf (required)"
|
| 109 |
+
@echo " HF_TOKEN - HF token (env var, required for deploy-hf)"
|
| 110 |
+
@echo ""
|
| 111 |
+
@echo "Examples:"
|
| 112 |
+
@echo " make deploy-dev SSH_TARGET=cloud.usa2"
|
| 113 |
+
@echo " make deploy-hf HF_SPACE=oaklight/veilrender"
|
|
@@ -1 +0,0 @@
|
|
| 1 |
-
# veilrender
|
|
|
|
|
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
README_en.md
|
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# VeilRender
|
| 2 |
+
|
| 3 |
+
[中文](README_zh.md) | **English**
|
| 4 |
+
|
| 5 |
+
Headless browser rendering API — self-hostable on HF Spaces, Docker, or bare metal.
|
| 6 |
+
|
| 7 |
+
VeilRender accepts a URL and returns the fully rendered page content (HTML, Markdown, readability-extracted article) using a headless Chromium browser. Designed as a fallback for fetch tools that fail on JavaScript-rendered pages.
|
| 8 |
+
|
| 9 |
+
## Quick Start
|
| 10 |
+
|
| 11 |
+
### Docker
|
| 12 |
+
|
| 13 |
+
```bash
|
| 14 |
+
docker run -p 7860:7860 -e VEILRENDER_API_TOKEN=your-secret ghcr.io/oaklight/veilrender
|
| 15 |
+
```
|
| 16 |
+
|
| 17 |
+
### Local Development
|
| 18 |
+
|
| 19 |
+
```bash
|
| 20 |
+
pip install -e ".[dev]"
|
| 21 |
+
playwright install chromium
|
| 22 |
+
python -m veilrender
|
| 23 |
+
```
|
| 24 |
+
|
| 25 |
+
## API
|
| 26 |
+
|
| 27 |
+
### GET /health
|
| 28 |
+
|
| 29 |
+
Returns `{"status": "ok"}` if the service is running.
|
| 30 |
+
|
| 31 |
+
### POST /render
|
| 32 |
+
|
| 33 |
+
Render a URL and return the page content.
|
| 34 |
+
|
| 35 |
+
```bash
|
| 36 |
+
curl -X POST http://localhost:7860/render \
|
| 37 |
+
-H "Authorization: Bearer your-secret" \
|
| 38 |
+
-H "Content-Type: application/json" \
|
| 39 |
+
-d '{"url": "https://example.com"}'
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
Response:
|
| 43 |
+
|
| 44 |
+
```json
|
| 45 |
+
{
|
| 46 |
+
"content": {
|
| 47 |
+
"html": "...",
|
| 48 |
+
"markdown": "...",
|
| 49 |
+
"readability": "..."
|
| 50 |
+
},
|
| 51 |
+
"metadata": {
|
| 52 |
+
"title": "Example Domain",
|
| 53 |
+
"url": "https://example.com",
|
| 54 |
+
"status_code": 200
|
| 55 |
+
},
|
| 56 |
+
"links": [{"url": "https://www.iana.org/domains/example", "text": "More information..."}]
|
| 57 |
+
}
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
### POST /screenshot
|
| 61 |
+
|
| 62 |
+
Capture a screenshot of a URL.
|
| 63 |
+
|
| 64 |
+
```bash
|
| 65 |
+
curl -X POST http://localhost:7860/screenshot \
|
| 66 |
+
-H "Authorization: Bearer your-secret" \
|
| 67 |
+
-H "Content-Type: application/json" \
|
| 68 |
+
-d '{"url": "https://example.com"}' \
|
| 69 |
+
-o screenshot.png
|
| 70 |
+
```
|
| 71 |
+
|
| 72 |
+
## Configuration
|
| 73 |
+
|
| 74 |
+
All settings are configured via environment variables with the `VEILRENDER_` prefix:
|
| 75 |
+
|
| 76 |
+
| Variable | Default | Description |
|
| 77 |
+
|----------|---------|-------------|
|
| 78 |
+
| `VEILRENDER_API_TOKEN` | *(none)* | API token for authentication. If unset, auth is disabled. |
|
| 79 |
+
| `VEILRENDER_PORT` | `7860` | Server port |
|
| 80 |
+
| `VEILRENDER_HOST` | `0.0.0.0` | Server bind address |
|
| 81 |
+
| `VEILRENDER_TIMEOUT` | `30000` | Browser navigation timeout (ms) |
|
| 82 |
+
| `VEILRENDER_VIEWPORT_WIDTH` | `1280` | Browser viewport width |
|
| 83 |
+
| `VEILRENDER_VIEWPORT_HEIGHT` | `720` | Browser viewport height |
|
| 84 |
+
| `VEILRENDER_MAX_CONCURRENT` | `3` | Max concurrent browser contexts |
|
| 85 |
+
|
| 86 |
+
## License
|
| 87 |
+
|
| 88 |
+
MIT
|
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# VeilRender
|
| 2 |
+
|
| 3 |
+
**中文** | [English](README_en.md)
|
| 4 |
+
|
| 5 |
+
无头浏览器渲染 API——可自托管于 HF Spaces、Docker 或物理机。
|
| 6 |
+
|
| 7 |
+
VeilRender 接收一个 URL,使用无头 Chromium 浏览器渲染页面,并返回完整内容(HTML、Markdown、readability 提取的文章)。专为 JavaScript 渲染页面的 fetch 降级方案设计。
|
| 8 |
+
|
| 9 |
+
## 快速开始
|
| 10 |
+
|
| 11 |
+
### Docker
|
| 12 |
+
|
| 13 |
+
```bash
|
| 14 |
+
docker run -p 7860:7860 -e VEILRENDER_API_TOKEN=your-secret ghcr.io/oaklight/veilrender
|
| 15 |
+
```
|
| 16 |
+
|
| 17 |
+
### 本地开发
|
| 18 |
+
|
| 19 |
+
```bash
|
| 20 |
+
pip install -e ".[dev]"
|
| 21 |
+
playwright install chromium
|
| 22 |
+
python -m veilrender
|
| 23 |
+
```
|
| 24 |
+
|
| 25 |
+
## API
|
| 26 |
+
|
| 27 |
+
### GET /health
|
| 28 |
+
|
| 29 |
+
服务运行中返回 `{"status": "ok"}`。
|
| 30 |
+
|
| 31 |
+
### POST /render
|
| 32 |
+
|
| 33 |
+
渲染指定 URL 并返回页面内容。
|
| 34 |
+
|
| 35 |
+
```bash
|
| 36 |
+
curl -X POST http://localhost:7860/render \
|
| 37 |
+
-H "Authorization: Bearer your-secret" \
|
| 38 |
+
-H "Content-Type: application/json" \
|
| 39 |
+
-d '{"url": "https://example.com"}'
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
响应:
|
| 43 |
+
|
| 44 |
+
```json
|
| 45 |
+
{
|
| 46 |
+
"content": {
|
| 47 |
+
"html": "...",
|
| 48 |
+
"markdown": "...",
|
| 49 |
+
"readability": "..."
|
| 50 |
+
},
|
| 51 |
+
"metadata": {
|
| 52 |
+
"title": "Example Domain",
|
| 53 |
+
"url": "https://example.com",
|
| 54 |
+
"status_code": 200
|
| 55 |
+
},
|
| 56 |
+
"links": [{"url": "https://www.iana.org/domains/example", "text": "More information..."}]
|
| 57 |
+
}
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
### POST /screenshot
|
| 61 |
+
|
| 62 |
+
截取指定 URL 的页面截图。
|
| 63 |
+
|
| 64 |
+
```bash
|
| 65 |
+
curl -X POST http://localhost:7860/screenshot \
|
| 66 |
+
-H "Authorization: Bearer your-secret" \
|
| 67 |
+
-H "Content-Type: application/json" \
|
| 68 |
+
-d '{"url": "https://example.com"}' \
|
| 69 |
+
-o screenshot.png
|
| 70 |
+
```
|
| 71 |
+
|
| 72 |
+
## 配置
|
| 73 |
+
|
| 74 |
+
所有设置通过 `VEILRENDER_` 前缀的环境变量配置:
|
| 75 |
+
|
| 76 |
+
| 变量 | 默认值 | 说明 |
|
| 77 |
+
|------|--------|------|
|
| 78 |
+
| `VEILRENDER_API_TOKEN` | *(无)* | API 认证令牌。未设置时认证关闭。 |
|
| 79 |
+
| `VEILRENDER_PORT` | `7860` | 服务端口 |
|
| 80 |
+
| `VEILRENDER_HOST` | `0.0.0.0` | 服务绑定地址 |
|
| 81 |
+
| `VEILRENDER_TIMEOUT` | `30000` | 浏览器导航超时(毫秒) |
|
| 82 |
+
| `VEILRENDER_VIEWPORT_WIDTH` | `1280` | 浏览器视口宽度 |
|
| 83 |
+
| `VEILRENDER_VIEWPORT_HEIGHT` | `720` | 浏览器视口高度 |
|
| 84 |
+
| `VEILRENDER_MAX_CONCURRENT` | `3` | 最大并发浏览器上下文数 |
|
| 85 |
+
|
| 86 |
+
## 许可证
|
| 87 |
+
|
| 88 |
+
MIT
|
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=61.0"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "veilrender"
|
| 7 |
+
dynamic = ["version"]
|
| 8 |
+
description = "Headless browser rendering API — self-hostable on HF Spaces, Docker, or bare metal"
|
| 9 |
+
authors = [{ name = "Peng Ding" }]
|
| 10 |
+
readme = "README.md"
|
| 11 |
+
requires-python = ">=3.10"
|
| 12 |
+
license = "MIT"
|
| 13 |
+
classifiers = [
|
| 14 |
+
"Intended Audience :: Developers",
|
| 15 |
+
"Programming Language :: Python :: 3",
|
| 16 |
+
"Topic :: Internet :: WWW/HTTP",
|
| 17 |
+
]
|
| 18 |
+
|
| 19 |
+
dependencies = [
|
| 20 |
+
"cloakbrowser>=0.3.0",
|
| 21 |
+
]
|
| 22 |
+
|
| 23 |
+
[project.optional-dependencies]
|
| 24 |
+
dev = [
|
| 25 |
+
"ruff>=0.11.0",
|
| 26 |
+
"ty>=0.0.1a0",
|
| 27 |
+
"pytest>=7.0.0",
|
| 28 |
+
"pytest-asyncio>=0.21.0",
|
| 29 |
+
]
|
| 30 |
+
|
| 31 |
+
[project.scripts]
|
| 32 |
+
veilrender = "veilrender.app:main"
|
| 33 |
+
|
| 34 |
+
[project.urls]
|
| 35 |
+
Repository = "https://github.com/Oaklight/veilrender"
|
| 36 |
+
Issues = "https://github.com/Oaklight/veilrender/issues"
|
| 37 |
+
|
| 38 |
+
[tool.setuptools.packages.find]
|
| 39 |
+
where = ["src"]
|
| 40 |
+
|
| 41 |
+
[tool.setuptools.dynamic]
|
| 42 |
+
version = { attr = "veilrender.__version__" }
|
| 43 |
+
|
| 44 |
+
[tool.setuptools.package-data]
|
| 45 |
+
"veilrender" = ["py.typed"]
|
| 46 |
+
|
| 47 |
+
[tool.ruff]
|
| 48 |
+
target-version = "py310"
|
| 49 |
+
|
| 50 |
+
[tool.ruff.lint]
|
| 51 |
+
select = ["E", "F", "UP"]
|
| 52 |
+
ignore = ["UP007", "E501"]
|
| 53 |
+
|
| 54 |
+
[tool.ty.environment]
|
| 55 |
+
python-version = "3.10"
|
| 56 |
+
|
| 57 |
+
[tool.ty.src]
|
| 58 |
+
exclude = ["src/veilrender/_vendor/**"]
|
| 59 |
+
|
| 60 |
+
[tool.ty.rules]
|
| 61 |
+
unresolved-import = "ignore"
|
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""VeilRender — headless browser rendering API."""
|
| 2 |
+
|
| 3 |
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Entry point for ``python -m veilrender``."""
|
| 2 |
+
|
| 3 |
+
from veilrender.app import main
|
| 4 |
+
|
| 5 |
+
if __name__ == "__main__":
|
| 6 |
+
main()
|
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Vendored zerodep modules — do not edit manually.
|
| 2 |
+
# Update via: make vendor
|
|
@@ -0,0 +1,323 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Three-way readability benchmark: zerodep vs readability-lxml vs Mozilla JS.
|
| 3 |
+
|
| 4 |
+
Runs each implementation on Mozilla's test fixtures and prints a comparison
|
| 5 |
+
table. JS timing is measured internally by bench_mozilla.js (no subprocess
|
| 6 |
+
overhead in the numbers).
|
| 7 |
+
|
| 8 |
+
Usage:
|
| 9 |
+
python benchmark_compare.py # all fixtures
|
| 10 |
+
python benchmark_compare.py 001 bbc-1 # specific fixtures
|
| 11 |
+
python benchmark_compare.py --rounds 20 # more rounds
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import argparse
|
| 17 |
+
import json
|
| 18 |
+
import os
|
| 19 |
+
import shutil
|
| 20 |
+
import subprocess
|
| 21 |
+
import sys
|
| 22 |
+
import timeit
|
| 23 |
+
|
| 24 |
+
# ── Setup paths ──────────────────────────────────────────────────────────────
|
| 25 |
+
|
| 26 |
+
_THIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 27 |
+
_TEST_PAGES_DIR = os.path.join(_THIS_DIR, "test-pages")
|
| 28 |
+
_BENCH_JS = os.path.join(_THIS_DIR, "bench_mozilla.js")
|
| 29 |
+
|
| 30 |
+
sys.path.insert(0, _THIS_DIR)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
# ── Discover fixtures ────────────────────────────────────────────────────────
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def discover_fixtures() -> list[str]:
|
| 37 |
+
"""Return sorted list of available fixture names."""
|
| 38 |
+
if not os.path.isdir(_TEST_PAGES_DIR):
|
| 39 |
+
return []
|
| 40 |
+
return sorted(
|
| 41 |
+
d
|
| 42 |
+
for d in os.listdir(_TEST_PAGES_DIR)
|
| 43 |
+
if os.path.isdir(os.path.join(_TEST_PAGES_DIR, d))
|
| 44 |
+
and os.path.isfile(os.path.join(_TEST_PAGES_DIR, d, "source.html"))
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def load_source(name: str) -> str:
|
| 49 |
+
"""Load source HTML for a fixture."""
|
| 50 |
+
path = os.path.join(_TEST_PAGES_DIR, name, "source.html")
|
| 51 |
+
with open(path, encoding="utf-8") as f:
|
| 52 |
+
return f.read()
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
# ── Python: zerodep readability ──────────────────────────────────────────────
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def bench_zerodep(html: str, rounds: int) -> dict:
|
| 59 |
+
"""Benchmark our readability.extract() and return timing dict."""
|
| 60 |
+
from readability import extract
|
| 61 |
+
|
| 62 |
+
# Warm-up.
|
| 63 |
+
result = extract(html)
|
| 64 |
+
|
| 65 |
+
times = []
|
| 66 |
+
for _ in range(rounds):
|
| 67 |
+
t0 = timeit.default_timer()
|
| 68 |
+
extract(html)
|
| 69 |
+
t1 = timeit.default_timer()
|
| 70 |
+
times.append((t1 - t0) * 1000) # ms
|
| 71 |
+
|
| 72 |
+
return {
|
| 73 |
+
"times_ms": times,
|
| 74 |
+
"min_ms": min(times),
|
| 75 |
+
"mean_ms": sum(times) / len(times),
|
| 76 |
+
"max_ms": max(times),
|
| 77 |
+
"title": result.title,
|
| 78 |
+
"length": result.length,
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
# ── Python: readability-lxml ────────────────────────────────────────────────
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _load_readability_lxml():
|
| 86 |
+
"""Load readability-lxml's Document class, working around name clash."""
|
| 87 |
+
import importlib
|
| 88 |
+
import importlib.metadata
|
| 89 |
+
|
| 90 |
+
try:
|
| 91 |
+
importlib.metadata.version("readability-lxml")
|
| 92 |
+
except importlib.metadata.PackageNotFoundError:
|
| 93 |
+
return None
|
| 94 |
+
|
| 95 |
+
saved_path = sys.path[:]
|
| 96 |
+
saved_modules = {
|
| 97 |
+
k: sys.modules.pop(k)
|
| 98 |
+
for k in list(sys.modules)
|
| 99 |
+
if k == "readability" or k.startswith("readability.")
|
| 100 |
+
}
|
| 101 |
+
try:
|
| 102 |
+
sys.path = [
|
| 103 |
+
p for p in sys.path if os.path.abspath(p) != os.path.abspath(_THIS_DIR)
|
| 104 |
+
]
|
| 105 |
+
mod = importlib.import_module("readability")
|
| 106 |
+
return mod.Document
|
| 107 |
+
finally:
|
| 108 |
+
sys.path = saved_path
|
| 109 |
+
for k in list(sys.modules):
|
| 110 |
+
if k == "readability" or k.startswith("readability."):
|
| 111 |
+
del sys.modules[k]
|
| 112 |
+
sys.modules.update(saved_modules)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
_RefDocument = _load_readability_lxml()
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def bench_readability_lxml(html: str, rounds: int) -> dict | None:
|
| 119 |
+
"""Benchmark readability-lxml and return timing dict, or None."""
|
| 120 |
+
if _RefDocument is None:
|
| 121 |
+
return None
|
| 122 |
+
|
| 123 |
+
# Warm-up.
|
| 124 |
+
doc = _RefDocument(html)
|
| 125 |
+
summary = doc.summary()
|
| 126 |
+
|
| 127 |
+
times = []
|
| 128 |
+
for _ in range(rounds):
|
| 129 |
+
t0 = timeit.default_timer()
|
| 130 |
+
doc = _RefDocument(html)
|
| 131 |
+
doc.summary()
|
| 132 |
+
t1 = timeit.default_timer()
|
| 133 |
+
times.append((t1 - t0) * 1000)
|
| 134 |
+
|
| 135 |
+
# Extract title from summary HTML (basic).
|
| 136 |
+
title = doc.short_title() if hasattr(doc, "short_title") else ""
|
| 137 |
+
length = len(summary) if summary else 0
|
| 138 |
+
|
| 139 |
+
return {
|
| 140 |
+
"times_ms": times,
|
| 141 |
+
"min_ms": min(times),
|
| 142 |
+
"mean_ms": sum(times) / len(times),
|
| 143 |
+
"max_ms": max(times),
|
| 144 |
+
"title": title,
|
| 145 |
+
"length": length,
|
| 146 |
+
}
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
# ── JavaScript: Mozilla Readability.js ───────────────────────────────────────
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def bench_mozilla_js(fixture_name: str, rounds: int) -> dict | None:
|
| 153 |
+
"""Benchmark Mozilla Readability.js via Node.js subprocess.
|
| 154 |
+
|
| 155 |
+
Timing is measured internally by bench_mozilla.js — no subprocess
|
| 156 |
+
overhead in the reported numbers.
|
| 157 |
+
"""
|
| 158 |
+
if not shutil.which("node"):
|
| 159 |
+
return None
|
| 160 |
+
if not os.path.isfile(_BENCH_JS):
|
| 161 |
+
return None
|
| 162 |
+
|
| 163 |
+
source_path = os.path.join(_TEST_PAGES_DIR, fixture_name, "source.html")
|
| 164 |
+
try:
|
| 165 |
+
result = subprocess.run(
|
| 166 |
+
["node", _BENCH_JS, source_path, str(rounds)],
|
| 167 |
+
capture_output=True,
|
| 168 |
+
text=True,
|
| 169 |
+
timeout=120,
|
| 170 |
+
cwd=_THIS_DIR,
|
| 171 |
+
)
|
| 172 |
+
if result.returncode != 0:
|
| 173 |
+
print(f" [JS error: {result.stderr.strip()[:100]}]", file=sys.stderr)
|
| 174 |
+
return None
|
| 175 |
+
return json.loads(result.stdout)
|
| 176 |
+
except (subprocess.TimeoutExpired, json.JSONDecodeError, FileNotFoundError):
|
| 177 |
+
return None
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
# ── Output formatting ────────────────────────────────────────────────────────
|
| 181 |
+
|
| 182 |
+
# ANSI colors (disabled if not a terminal).
|
| 183 |
+
if sys.stdout.isatty():
|
| 184 |
+
_BOLD = "\033[1m"
|
| 185 |
+
_GREEN = "\033[32m"
|
| 186 |
+
_YELLOW = "\033[33m"
|
| 187 |
+
_CYAN = "\033[36m"
|
| 188 |
+
_RESET = "\033[0m"
|
| 189 |
+
_DIM = "\033[2m"
|
| 190 |
+
else:
|
| 191 |
+
_BOLD = _GREEN = _YELLOW = _CYAN = _RESET = _DIM = ""
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def _fmt_ms(ms: float) -> str:
|
| 195 |
+
"""Format milliseconds with appropriate unit."""
|
| 196 |
+
if ms < 1:
|
| 197 |
+
return f"{ms * 1000:.0f} µs"
|
| 198 |
+
if ms < 1000:
|
| 199 |
+
return f"{ms:.1f} ms"
|
| 200 |
+
return f"{ms / 1000:.2f} s"
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def _ratio_str(ms: float, baseline: float) -> str:
|
| 204 |
+
"""Format a ratio relative to baseline."""
|
| 205 |
+
if baseline <= 0:
|
| 206 |
+
return ""
|
| 207 |
+
ratio = ms / baseline
|
| 208 |
+
if ratio < 1.05:
|
| 209 |
+
return f"{_GREEN}1.00x{_RESET}"
|
| 210 |
+
return f"{_YELLOW}{ratio:.2f}x{_RESET}"
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def print_results(
|
| 214 |
+
fixture_name: str,
|
| 215 |
+
html_size: int,
|
| 216 |
+
zd: dict,
|
| 217 |
+
lxml: dict | None,
|
| 218 |
+
js: dict | None,
|
| 219 |
+
) -> None:
|
| 220 |
+
"""Print a single fixture's results as a formatted row."""
|
| 221 |
+
baseline = zd["mean_ms"]
|
| 222 |
+
|
| 223 |
+
cols = [
|
| 224 |
+
f" {_BOLD}{fixture_name:<28s}{_RESET}",
|
| 225 |
+
f"{_DIM}{html_size / 1024:>7.1f} KB{_RESET}",
|
| 226 |
+
f"{_CYAN}zerodep{_RESET} {_fmt_ms(zd['mean_ms']):>10s}"
|
| 227 |
+
f" {_ratio_str(zd['mean_ms'], baseline)}",
|
| 228 |
+
]
|
| 229 |
+
|
| 230 |
+
if lxml is not None:
|
| 231 |
+
cols.append(
|
| 232 |
+
f"{_CYAN}lxml{_RESET} {_fmt_ms(lxml['mean_ms']):>10s}"
|
| 233 |
+
f" {_ratio_str(lxml['mean_ms'], baseline)}"
|
| 234 |
+
)
|
| 235 |
+
else:
|
| 236 |
+
cols.append(f"{_DIM}lxml {'n/a':>10s}{_RESET}")
|
| 237 |
+
|
| 238 |
+
if js is not None:
|
| 239 |
+
cols.append(
|
| 240 |
+
f"{_CYAN}mozilla{_RESET} {_fmt_ms(js['mean_ms']):>10s}"
|
| 241 |
+
f" {_ratio_str(js['mean_ms'], baseline)}"
|
| 242 |
+
)
|
| 243 |
+
else:
|
| 244 |
+
cols.append(f"{_DIM}mozilla {'n/a':>10s}{_RESET}")
|
| 245 |
+
|
| 246 |
+
print(" ".join(cols))
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
# ── Main ─────────────────────────────────────────────────────────────────────
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def main() -> None:
|
| 253 |
+
parser = argparse.ArgumentParser(
|
| 254 |
+
description="Three-way readability benchmark comparison."
|
| 255 |
+
)
|
| 256 |
+
parser.add_argument(
|
| 257 |
+
"fixtures",
|
| 258 |
+
nargs="*",
|
| 259 |
+
help="Fixture names to benchmark (default: all).",
|
| 260 |
+
)
|
| 261 |
+
parser.add_argument(
|
| 262 |
+
"--rounds",
|
| 263 |
+
type=int,
|
| 264 |
+
default=10,
|
| 265 |
+
help="Number of timing rounds per fixture (default: 10).",
|
| 266 |
+
)
|
| 267 |
+
args = parser.parse_args()
|
| 268 |
+
|
| 269 |
+
all_fixtures = discover_fixtures()
|
| 270 |
+
if not all_fixtures:
|
| 271 |
+
print("No test fixtures found in test-pages/", file=sys.stderr)
|
| 272 |
+
sys.exit(1)
|
| 273 |
+
|
| 274 |
+
fixtures = args.fixtures if args.fixtures else all_fixtures
|
| 275 |
+
# Validate fixture names.
|
| 276 |
+
for name in fixtures:
|
| 277 |
+
if name not in all_fixtures:
|
| 278 |
+
print(f"Unknown fixture: {name}", file=sys.stderr)
|
| 279 |
+
print(f"Available: {', '.join(all_fixtures)}", file=sys.stderr)
|
| 280 |
+
sys.exit(1)
|
| 281 |
+
|
| 282 |
+
rounds = args.rounds
|
| 283 |
+
|
| 284 |
+
# Header.
|
| 285 |
+
print()
|
| 286 |
+
print(f"{_BOLD}Readability Benchmark ({rounds} rounds per fixture){_RESET}")
|
| 287 |
+
has_node = shutil.which("node") is not None
|
| 288 |
+
has_lxml = _RefDocument is not None
|
| 289 |
+
status = []
|
| 290 |
+
status.append(f"zerodep: {_GREEN}yes{_RESET}")
|
| 291 |
+
lxml_status = _GREEN + "yes" + _RESET if has_lxml else _DIM + "no" + _RESET
|
| 292 |
+
status.append(f"readability-lxml: {lxml_status}")
|
| 293 |
+
status.append(
|
| 294 |
+
f"mozilla js: {_GREEN + 'yes' + _RESET if has_node else _DIM + 'no' + _RESET}"
|
| 295 |
+
)
|
| 296 |
+
print(f" Implementations: {' | '.join(status)}")
|
| 297 |
+
print(f" {_DIM}Times shown are mean. Ratios relative to zerodep.{_RESET}")
|
| 298 |
+
print()
|
| 299 |
+
|
| 300 |
+
# Column headers.
|
| 301 |
+
print(
|
| 302 |
+
f" {'Fixture':<28s} {'Size':>9s} "
|
| 303 |
+
f"{'zerodep':>19s} {'readability-lxml':>19s} "
|
| 304 |
+
f"{'mozilla js':>19s}"
|
| 305 |
+
)
|
| 306 |
+
print(" " + "─" * 110)
|
| 307 |
+
|
| 308 |
+
for name in fixtures:
|
| 309 |
+
html = load_source(name)
|
| 310 |
+
html_size = len(html.encode("utf-8"))
|
| 311 |
+
|
| 312 |
+
# Benchmark all three.
|
| 313 |
+
zd = bench_zerodep(html, rounds)
|
| 314 |
+
lxml = bench_readability_lxml(html, rounds)
|
| 315 |
+
js = bench_mozilla_js(name, rounds)
|
| 316 |
+
|
| 317 |
+
print_results(name, html_size, zd, lxml, js)
|
| 318 |
+
|
| 319 |
+
print()
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
if __name__ == "__main__":
|
| 323 |
+
main()
|
|
@@ -0,0 +1,1023 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.2.4"
|
| 3 |
+
# deps = []
|
| 4 |
+
# tier = "subsystem"
|
| 5 |
+
# category = "storage"
|
| 6 |
+
# note = "Install/update via `zerodep add cache`"
|
| 7 |
+
# ///
|
| 8 |
+
"""Zero-dependency caching with TTL, eviction policies, and async support.
|
| 9 |
+
|
| 10 |
+
stdlib only, Python 3.10+.
|
| 11 |
+
|
| 12 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 13 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 14 |
+
|
| 15 |
+
Provides: LRUCache, FIFOCache, LFUCache, TTLCache cache classes (MutableMapping),
|
| 16 |
+
``cached`` decorator with sync/async auto-detection, and convenience decorators
|
| 17 |
+
``lru_cache``, ``ttl_cache``, ``lfu_cache``, ``fifo_cache``.
|
| 18 |
+
|
| 19 |
+
Does NOT implement: TLRUCache (per-item TTL function), MRUCache, RRCache,
|
| 20 |
+
``cachedmethod``, ``condition`` parameter, multiple backends, serialization.
|
| 21 |
+
|
| 22 |
+
Example::
|
| 23 |
+
|
| 24 |
+
from cache import lru_cache, ttl_cache, LRUCache, cached
|
| 25 |
+
|
| 26 |
+
# Convenience decorator (like functools.lru_cache but with more policies)
|
| 27 |
+
@lru_cache(maxsize=256)
|
| 28 |
+
def fibonacci(n):
|
| 29 |
+
return n if n < 2 else fibonacci(n - 1) + fibonacci(n - 2)
|
| 30 |
+
|
| 31 |
+
# TTL decorator
|
| 32 |
+
@ttl_cache(maxsize=100, ttl=60)
|
| 33 |
+
def fetch_config(key):
|
| 34 |
+
return load_from_db(key)
|
| 35 |
+
|
| 36 |
+
# Async support (auto-detected)
|
| 37 |
+
@lru_cache(maxsize=128)
|
| 38 |
+
async def fetch_data(url):
|
| 39 |
+
return await async_get(url)
|
| 40 |
+
|
| 41 |
+
# Direct cache class usage
|
| 42 |
+
cache = LRUCache(maxsize=1024)
|
| 43 |
+
cache["key"] = "value"
|
| 44 |
+
|
| 45 |
+
# Advanced: cached() with explicit lock
|
| 46 |
+
import threading
|
| 47 |
+
@cached(LRUCache(128), lock=threading.Lock(), info=True)
|
| 48 |
+
def thread_safe_compute(x):
|
| 49 |
+
return expensive(x)
|
| 50 |
+
"""
|
| 51 |
+
|
| 52 |
+
from __future__ import annotations
|
| 53 |
+
|
| 54 |
+
import collections
|
| 55 |
+
import collections.abc
|
| 56 |
+
import functools
|
| 57 |
+
import inspect
|
| 58 |
+
import time
|
| 59 |
+
from typing import Any, Callable
|
| 60 |
+
|
| 61 |
+
# ── Key Functions ──────────────────────────────────────────────────────────
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class _HashedTuple(tuple):
|
| 65 |
+
"""Tuple subclass that caches its hash value for fast repeated lookups."""
|
| 66 |
+
|
| 67 |
+
__hashvalue: int | None = None
|
| 68 |
+
|
| 69 |
+
def __hash__(self, hash=tuple.__hash__): # type: ignore[override]
|
| 70 |
+
hashvalue = self.__hashvalue
|
| 71 |
+
if hashvalue is None:
|
| 72 |
+
self.__hashvalue = hashvalue = hash(self)
|
| 73 |
+
return hashvalue
|
| 74 |
+
|
| 75 |
+
def __add__(self, other, add=tuple.__add__):
|
| 76 |
+
return _HashedTuple(add(self, other))
|
| 77 |
+
|
| 78 |
+
def __radd__(self, other, add=tuple.__add__):
|
| 79 |
+
return _HashedTuple(add(other, self))
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
_KWMARK = (_HashedTuple,)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def hashkey(*args: Any, **kwargs: Any) -> _HashedTuple:
|
| 86 |
+
"""Return a cache key for the given positional and keyword arguments.
|
| 87 |
+
|
| 88 |
+
This is the default key function for all cache decorators.
|
| 89 |
+
"""
|
| 90 |
+
if kwargs:
|
| 91 |
+
return _HashedTuple(args + _KWMARK + tuple(sorted(kwargs.items())))
|
| 92 |
+
return _HashedTuple(args)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def methodkey(_self: Any, *args: Any, **kwargs: Any) -> _HashedTuple:
|
| 96 |
+
"""Like ``hashkey`` but ignores the first positional argument (``self``).
|
| 97 |
+
|
| 98 |
+
Use as the key function when caching instance methods.
|
| 99 |
+
"""
|
| 100 |
+
return hashkey(*args, **kwargs)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def typedkey(*args: Any, **kwargs: Any) -> _HashedTuple:
|
| 104 |
+
"""Like ``hashkey`` but includes type information, so ``f(3)`` and
|
| 105 |
+
``f(3.0)`` are cached separately.
|
| 106 |
+
"""
|
| 107 |
+
if kwargs:
|
| 108 |
+
sorted_items = tuple(sorted(kwargs.items()))
|
| 109 |
+
return _HashedTuple(
|
| 110 |
+
args
|
| 111 |
+
+ _KWMARK
|
| 112 |
+
+ sorted_items
|
| 113 |
+
+ tuple(type(v) for v in args)
|
| 114 |
+
+ tuple(type(v) for _, v in sorted_items)
|
| 115 |
+
)
|
| 116 |
+
return _HashedTuple(args + tuple(type(v) for v in args))
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
# ── Cache Info ─────────────────────────────────────────────────────────────
|
| 120 |
+
|
| 121 |
+
CacheInfo = collections.namedtuple(
|
| 122 |
+
"CacheInfo", ["hits", "misses", "maxsize", "currsize"]
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
# ── Cache Base Class ───────────────────────────────────────────────────────
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
class _DefaultSize:
|
| 129 |
+
"""Lightweight size tracker that always returns 1 for any key."""
|
| 130 |
+
|
| 131 |
+
__slots__ = ()
|
| 132 |
+
|
| 133 |
+
def __getitem__(self, _: Any) -> int:
|
| 134 |
+
return 1
|
| 135 |
+
|
| 136 |
+
def __setitem__(self, _: Any, __: Any) -> None:
|
| 137 |
+
pass
|
| 138 |
+
|
| 139 |
+
def pop(self, _: Any, *args: Any) -> int:
|
| 140 |
+
return 1
|
| 141 |
+
|
| 142 |
+
def clear(self) -> None:
|
| 143 |
+
pass
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
class Cache(collections.abc.MutableMapping):
|
| 147 |
+
"""Mutable mapping to serve as a simple cache or cache base class.
|
| 148 |
+
|
| 149 |
+
Subclasses must override ``popitem`` to implement a specific eviction
|
| 150 |
+
policy. The base class evicts arbitrary items (dict order).
|
| 151 |
+
|
| 152 |
+
Args:
|
| 153 |
+
maxsize: Maximum capacity (item count or weighted size).
|
| 154 |
+
getsizeof: Optional callable returning the size of a value.
|
| 155 |
+
Defaults to 1 per item.
|
| 156 |
+
"""
|
| 157 |
+
|
| 158 |
+
__marker = object()
|
| 159 |
+
|
| 160 |
+
def __init__(
|
| 161 |
+
self,
|
| 162 |
+
maxsize: int | float,
|
| 163 |
+
getsizeof: Callable[[Any], int] | None = None,
|
| 164 |
+
):
|
| 165 |
+
if maxsize < 0:
|
| 166 |
+
raise ValueError("maxsize must be non-negative")
|
| 167 |
+
self.__data: dict[Any, Any] = {}
|
| 168 |
+
self.__currsize: int | float = 0
|
| 169 |
+
self.__maxsize = maxsize
|
| 170 |
+
if getsizeof is not None:
|
| 171 |
+
self.__size: dict | _DefaultSize = {}
|
| 172 |
+
self.getsizeof = getsizeof # type: ignore[assignment]
|
| 173 |
+
else:
|
| 174 |
+
self.__size = _DefaultSize()
|
| 175 |
+
|
| 176 |
+
def __repr__(self) -> str:
|
| 177 |
+
cls = self.__class__.__name__
|
| 178 |
+
return (
|
| 179 |
+
f"{cls}({self.__data!r}, "
|
| 180 |
+
f"maxsize={self.__maxsize}, currsize={self.__currsize})"
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
def __getitem__(self, key: Any) -> Any:
|
| 184 |
+
try:
|
| 185 |
+
return self.__data[key]
|
| 186 |
+
except KeyError:
|
| 187 |
+
return self.__missing__(key)
|
| 188 |
+
|
| 189 |
+
def __setitem__(self, key: Any, value: Any) -> None:
|
| 190 |
+
maxsize = self.__maxsize
|
| 191 |
+
size = self.getsizeof(value)
|
| 192 |
+
if size > maxsize:
|
| 193 |
+
raise ValueError("value too large")
|
| 194 |
+
if key in self.__data:
|
| 195 |
+
diffsize = size - self.__size[key]
|
| 196 |
+
else:
|
| 197 |
+
diffsize = size
|
| 198 |
+
while self.__currsize + diffsize > maxsize:
|
| 199 |
+
self.popitem()
|
| 200 |
+
if key in self.__data:
|
| 201 |
+
# Key might have been evicted during popitem loop
|
| 202 |
+
self.__currsize -= self.__size[key]
|
| 203 |
+
self.__data[key] = value
|
| 204 |
+
self.__size[key] = size
|
| 205 |
+
self.__currsize += size
|
| 206 |
+
|
| 207 |
+
def __delitem__(self, key: Any) -> None:
|
| 208 |
+
size = self.__size.pop(key)
|
| 209 |
+
del self.__data[key]
|
| 210 |
+
self.__currsize -= size
|
| 211 |
+
|
| 212 |
+
def __contains__(self, key: object) -> bool:
|
| 213 |
+
return key in self.__data
|
| 214 |
+
|
| 215 |
+
def __missing__(self, key: Any) -> Any:
|
| 216 |
+
raise KeyError(key)
|
| 217 |
+
|
| 218 |
+
def __iter__(self):
|
| 219 |
+
return iter(self.__data)
|
| 220 |
+
|
| 221 |
+
def __len__(self) -> int:
|
| 222 |
+
return len(self.__data)
|
| 223 |
+
|
| 224 |
+
@staticmethod
|
| 225 |
+
def getsizeof(value: Any) -> int:
|
| 226 |
+
"""Return the size of *value*. Defaults to 1."""
|
| 227 |
+
return 1
|
| 228 |
+
|
| 229 |
+
@property
|
| 230 |
+
def maxsize(self) -> int | float:
|
| 231 |
+
"""Maximum cache capacity."""
|
| 232 |
+
return self.__maxsize
|
| 233 |
+
|
| 234 |
+
@property
|
| 235 |
+
def currsize(self) -> int | float:
|
| 236 |
+
"""Current cache size (sum of item sizes)."""
|
| 237 |
+
return self.__currsize
|
| 238 |
+
|
| 239 |
+
def get(self, key: Any, default: Any = None) -> Any:
|
| 240 |
+
if key in self:
|
| 241 |
+
return self[key]
|
| 242 |
+
return default
|
| 243 |
+
|
| 244 |
+
def pop(self, key: Any, default: Any = __marker) -> Any:
|
| 245 |
+
if key in self:
|
| 246 |
+
value = self[key]
|
| 247 |
+
del self[key]
|
| 248 |
+
return value
|
| 249 |
+
if default is self.__marker:
|
| 250 |
+
raise KeyError(key)
|
| 251 |
+
return default
|
| 252 |
+
|
| 253 |
+
def setdefault(self, key: Any, default: Any = None) -> Any:
|
| 254 |
+
if key in self:
|
| 255 |
+
return self[key]
|
| 256 |
+
try:
|
| 257 |
+
self[key] = default
|
| 258 |
+
except ValueError:
|
| 259 |
+
pass
|
| 260 |
+
return default
|
| 261 |
+
|
| 262 |
+
def popitem(self) -> tuple[Any, Any]:
|
| 263 |
+
"""Remove and return an arbitrary ``(key, value)`` pair.
|
| 264 |
+
|
| 265 |
+
Subclasses override this to implement eviction policies.
|
| 266 |
+
"""
|
| 267 |
+
try:
|
| 268 |
+
key = next(iter(self.__data))
|
| 269 |
+
except StopIteration:
|
| 270 |
+
raise KeyError(f"{type(self).__name__} is empty") from None
|
| 271 |
+
return key, self.pop(key)
|
| 272 |
+
|
| 273 |
+
def clear(self) -> None:
|
| 274 |
+
self.__data.clear()
|
| 275 |
+
self.__size.clear()
|
| 276 |
+
self.__currsize = 0
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
# ── FIFO Cache ─────────────────────────────────────────────────────────────
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
class FIFOCache(Cache):
|
| 283 |
+
"""First In First Out (FIFO) cache implementation.
|
| 284 |
+
|
| 285 |
+
Evicts the oldest inserted item when the cache is full. Accessing an
|
| 286 |
+
item does **not** change its eviction priority.
|
| 287 |
+
"""
|
| 288 |
+
|
| 289 |
+
def __init__(
|
| 290 |
+
self,
|
| 291 |
+
maxsize: int | float,
|
| 292 |
+
getsizeof: Callable[[Any], int] | None = None,
|
| 293 |
+
):
|
| 294 |
+
super().__init__(maxsize, getsizeof)
|
| 295 |
+
self.__order: collections.OrderedDict[Any, None] = collections.OrderedDict()
|
| 296 |
+
|
| 297 |
+
def __setitem__(self, key: Any, value: Any) -> None:
|
| 298 |
+
if key in self:
|
| 299 |
+
del self.__order[key]
|
| 300 |
+
super().__setitem__(key, value)
|
| 301 |
+
self.__order[key] = None
|
| 302 |
+
|
| 303 |
+
def __delitem__(self, key: Any) -> None:
|
| 304 |
+
super().__delitem__(key)
|
| 305 |
+
del self.__order[key]
|
| 306 |
+
|
| 307 |
+
def popitem(self) -> tuple[Any, Any]:
|
| 308 |
+
"""Remove and return the oldest inserted ``(key, value)`` pair."""
|
| 309 |
+
try:
|
| 310 |
+
key = next(iter(self.__order))
|
| 311 |
+
except StopIteration:
|
| 312 |
+
raise KeyError(f"{type(self).__name__} is empty") from None
|
| 313 |
+
return key, self.pop(key)
|
| 314 |
+
|
| 315 |
+
def clear(self) -> None:
|
| 316 |
+
super().clear()
|
| 317 |
+
self.__order.clear()
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
# ── LRU Cache ──────────────────────────────────────────────────────────────
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
class LRUCache(Cache):
|
| 324 |
+
"""Least Recently Used (LRU) cache implementation.
|
| 325 |
+
|
| 326 |
+
Evicts the least recently accessed item when the cache is full.
|
| 327 |
+
Both reads and writes count as accesses.
|
| 328 |
+
"""
|
| 329 |
+
|
| 330 |
+
def __init__(
|
| 331 |
+
self,
|
| 332 |
+
maxsize: int | float,
|
| 333 |
+
getsizeof: Callable[[Any], int] | None = None,
|
| 334 |
+
):
|
| 335 |
+
super().__init__(maxsize, getsizeof)
|
| 336 |
+
self.__order: collections.OrderedDict[Any, None] = collections.OrderedDict()
|
| 337 |
+
|
| 338 |
+
def __getitem__(self, key: Any) -> Any:
|
| 339 |
+
# Bypass super().__getitem__ to avoid MRO dispatch overhead;
|
| 340 |
+
# call Cache.__getitem__ directly + inline move_to_end.
|
| 341 |
+
value = Cache.__getitem__(self, key)
|
| 342 |
+
self.__order.move_to_end(key)
|
| 343 |
+
return value
|
| 344 |
+
|
| 345 |
+
def __setitem__(self, key: Any, value: Any) -> None:
|
| 346 |
+
Cache.__setitem__(self, key, value)
|
| 347 |
+
try:
|
| 348 |
+
self.__order.move_to_end(key)
|
| 349 |
+
except KeyError:
|
| 350 |
+
self.__order[key] = None
|
| 351 |
+
|
| 352 |
+
def __delitem__(self, key: Any) -> None:
|
| 353 |
+
Cache.__delitem__(self, key)
|
| 354 |
+
del self.__order[key]
|
| 355 |
+
|
| 356 |
+
def popitem(self) -> tuple[Any, Any]:
|
| 357 |
+
"""Remove and return the least recently used ``(key, value)`` pair."""
|
| 358 |
+
try:
|
| 359 |
+
key = next(iter(self.__order))
|
| 360 |
+
except StopIteration:
|
| 361 |
+
raise KeyError(f"{type(self).__name__} is empty") from None
|
| 362 |
+
return key, self.pop(key)
|
| 363 |
+
|
| 364 |
+
def clear(self) -> None:
|
| 365 |
+
Cache.clear(self)
|
| 366 |
+
self.__order.clear()
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
# ── LFU Cache ──────────────────────────────────────────────────────────────
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
class _LFUNode:
|
| 373 |
+
"""Doubly-linked node for the LFU frequency list."""
|
| 374 |
+
|
| 375 |
+
__slots__ = ("count", "keys", "prev", "next")
|
| 376 |
+
|
| 377 |
+
def __init__(self, count: int = 0):
|
| 378 |
+
self.count = count
|
| 379 |
+
self.keys: set[Any] = set()
|
| 380 |
+
self.prev: _LFUNode = self
|
| 381 |
+
self.next: _LFUNode = self
|
| 382 |
+
|
| 383 |
+
def insert_after(self, node: _LFUNode) -> None:
|
| 384 |
+
"""Insert *node* after this node."""
|
| 385 |
+
node.prev = self
|
| 386 |
+
node.next = self.next
|
| 387 |
+
self.next.prev = node
|
| 388 |
+
self.next = node
|
| 389 |
+
|
| 390 |
+
def unlink(self) -> None:
|
| 391 |
+
"""Remove this node from the linked list."""
|
| 392 |
+
self.prev.next = self.next
|
| 393 |
+
self.next.prev = self.prev
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
class LFUCache(Cache):
|
| 397 |
+
"""Least Frequently Used (LFU) cache implementation.
|
| 398 |
+
|
| 399 |
+
Evicts the least frequently accessed item when the cache is full.
|
| 400 |
+
All operations are O(1).
|
| 401 |
+
"""
|
| 402 |
+
|
| 403 |
+
def __init__(
|
| 404 |
+
self,
|
| 405 |
+
maxsize: int | float,
|
| 406 |
+
getsizeof: Callable[[Any], int] | None = None,
|
| 407 |
+
):
|
| 408 |
+
super().__init__(maxsize, getsizeof)
|
| 409 |
+
self.__root = _LFUNode() # sentinel
|
| 410 |
+
self.__links: dict[Any, _LFUNode] = {}
|
| 411 |
+
|
| 412 |
+
def __getitem__(self, key: Any) -> Any:
|
| 413 |
+
value = super().__getitem__(key)
|
| 414 |
+
self.__touch(key)
|
| 415 |
+
return value
|
| 416 |
+
|
| 417 |
+
def __setitem__(self, key: Any, value: Any) -> None:
|
| 418 |
+
existed = key in self
|
| 419 |
+
super().__setitem__(key, value)
|
| 420 |
+
if existed:
|
| 421 |
+
self.__touch(key)
|
| 422 |
+
else:
|
| 423 |
+
# New key starts at count=1
|
| 424 |
+
root = self.__root
|
| 425 |
+
first = root.next
|
| 426 |
+
if first is root or first.count != 1:
|
| 427 |
+
node = _LFUNode(count=1)
|
| 428 |
+
root.insert_after(node)
|
| 429 |
+
else:
|
| 430 |
+
node = first
|
| 431 |
+
node.keys.add(key)
|
| 432 |
+
self.__links[key] = node
|
| 433 |
+
|
| 434 |
+
def __delitem__(self, key: Any) -> None:
|
| 435 |
+
super().__delitem__(key)
|
| 436 |
+
node = self.__links.pop(key)
|
| 437 |
+
node.keys.discard(key)
|
| 438 |
+
if not node.keys:
|
| 439 |
+
node.unlink()
|
| 440 |
+
|
| 441 |
+
def __touch(self, key: Any) -> None:
|
| 442 |
+
"""Increment frequency count for *key*."""
|
| 443 |
+
node = self.__links[key]
|
| 444 |
+
new_count = node.count + 1
|
| 445 |
+
nxt = node.next
|
| 446 |
+
if nxt is self.__root or nxt.count != new_count:
|
| 447 |
+
new_node = _LFUNode(count=new_count)
|
| 448 |
+
node.insert_after(new_node)
|
| 449 |
+
else:
|
| 450 |
+
new_node = nxt
|
| 451 |
+
new_node.keys.add(key)
|
| 452 |
+
node.keys.discard(key)
|
| 453 |
+
self.__links[key] = new_node
|
| 454 |
+
if not node.keys:
|
| 455 |
+
node.unlink()
|
| 456 |
+
|
| 457 |
+
def popitem(self) -> tuple[Any, Any]:
|
| 458 |
+
"""Remove and return the least frequently used ``(key, value)`` pair."""
|
| 459 |
+
root = self.__root
|
| 460 |
+
node = root.next
|
| 461 |
+
if node is root:
|
| 462 |
+
raise KeyError(f"{type(self).__name__} is empty")
|
| 463 |
+
key = next(iter(node.keys))
|
| 464 |
+
# Bypass self.pop → self[key] → __touch to avoid wasteful
|
| 465 |
+
# frequency increment on a key that is about to be deleted.
|
| 466 |
+
value = Cache.__getitem__(self, key)
|
| 467 |
+
del self[key]
|
| 468 |
+
return key, value
|
| 469 |
+
|
| 470 |
+
def clear(self) -> None:
|
| 471 |
+
super().clear()
|
| 472 |
+
self.__links.clear()
|
| 473 |
+
self.__root.prev = self.__root
|
| 474 |
+
self.__root.next = self.__root
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
# ── TTL Cache ─────────────────────��────────────────────────────────────────
|
| 478 |
+
|
| 479 |
+
|
| 480 |
+
class _Timer:
|
| 481 |
+
"""Wraps a timer function with nestable time-freezing.
|
| 482 |
+
|
| 483 |
+
Used by TTLCache to ensure time consistency within a single operation.
|
| 484 |
+
"""
|
| 485 |
+
|
| 486 |
+
__slots__ = ("_timer", "_nesting", "_time")
|
| 487 |
+
|
| 488 |
+
def __init__(self, timer: Callable[[], float] = time.monotonic):
|
| 489 |
+
self._timer = timer
|
| 490 |
+
self._nesting = 0
|
| 491 |
+
self._time: float = 0.0
|
| 492 |
+
|
| 493 |
+
def __call__(self) -> float:
|
| 494 |
+
if self._nesting == 0:
|
| 495 |
+
return self._timer()
|
| 496 |
+
return self._time
|
| 497 |
+
|
| 498 |
+
def __enter__(self) -> float:
|
| 499 |
+
if self._nesting == 0:
|
| 500 |
+
self._time = self._timer()
|
| 501 |
+
self._nesting += 1
|
| 502 |
+
return self._time
|
| 503 |
+
|
| 504 |
+
def __exit__(self, *exc: Any) -> None:
|
| 505 |
+
self._nesting -= 1
|
| 506 |
+
|
| 507 |
+
|
| 508 |
+
class _TTLLink:
|
| 509 |
+
"""Doubly-linked node for TTLCache expiry ordering."""
|
| 510 |
+
|
| 511 |
+
__slots__ = ("key", "expires", "prev", "next")
|
| 512 |
+
|
| 513 |
+
def __init__(
|
| 514 |
+
self,
|
| 515 |
+
key: Any = None,
|
| 516 |
+
expires: float = 0.0,
|
| 517 |
+
):
|
| 518 |
+
self.key = key
|
| 519 |
+
self.expires = expires
|
| 520 |
+
self.prev: _TTLLink = self
|
| 521 |
+
self.next: _TTLLink = self
|
| 522 |
+
|
| 523 |
+
def unlink(self) -> None:
|
| 524 |
+
self.prev.next = self.next
|
| 525 |
+
self.next.prev = self.prev
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
class TTLCache(LRUCache):
|
| 529 |
+
"""LRU cache with per-item time-to-live (TTL).
|
| 530 |
+
|
| 531 |
+
Items are evicted when they expire (checked lazily on access) or when
|
| 532 |
+
the cache is full (LRU order).
|
| 533 |
+
|
| 534 |
+
Args:
|
| 535 |
+
maxsize: Maximum cache capacity.
|
| 536 |
+
ttl: Time-to-live in seconds for each item.
|
| 537 |
+
timer: Callable returning the current time (default: ``time.monotonic``).
|
| 538 |
+
getsizeof: Optional callable returning the size of a value.
|
| 539 |
+
"""
|
| 540 |
+
|
| 541 |
+
def __init__(
|
| 542 |
+
self,
|
| 543 |
+
maxsize: int | float,
|
| 544 |
+
ttl: float,
|
| 545 |
+
*,
|
| 546 |
+
timer: Callable[[], float] = time.monotonic,
|
| 547 |
+
getsizeof: Callable[[Any], int] | None = None,
|
| 548 |
+
):
|
| 549 |
+
LRUCache.__init__(self, maxsize, getsizeof)
|
| 550 |
+
self.__ttl = ttl
|
| 551 |
+
self.__timer = _Timer(timer)
|
| 552 |
+
self.__root = _TTLLink() # sentinel for expiry list
|
| 553 |
+
self.__links: dict[Any, _TTLLink] = {}
|
| 554 |
+
|
| 555 |
+
@property
|
| 556 |
+
def ttl(self) -> float:
|
| 557 |
+
"""Time-to-live in seconds."""
|
| 558 |
+
return self.__ttl
|
| 559 |
+
|
| 560 |
+
@property
|
| 561 |
+
def timer(self) -> _Timer:
|
| 562 |
+
"""The timer function."""
|
| 563 |
+
return self.__timer
|
| 564 |
+
|
| 565 |
+
def __contains__(self, key: object) -> bool:
|
| 566 |
+
link = self.__links.get(key)
|
| 567 |
+
if link is None:
|
| 568 |
+
return False
|
| 569 |
+
return self.__timer() < link.expires
|
| 570 |
+
|
| 571 |
+
def __getitem__(self, key: Any) -> Any:
|
| 572 |
+
link = self.__links.get(key)
|
| 573 |
+
if link is not None and self.__timer() >= link.expires:
|
| 574 |
+
del self[key]
|
| 575 |
+
return self.__missing__(key)
|
| 576 |
+
# Bypass LRUCache.__getitem__ → Cache.__getitem__ call chain;
|
| 577 |
+
# inline both to save one method-call and one dict lookup.
|
| 578 |
+
# Same direct-call pattern used by LFUCache.popitem.
|
| 579 |
+
value = Cache.__getitem__(self, key)
|
| 580 |
+
self._LRUCache__order.move_to_end(key) # type: ignore[attr-defined]
|
| 581 |
+
return value
|
| 582 |
+
|
| 583 |
+
def __setitem__(self, key: Any, value: Any) -> None:
|
| 584 |
+
with self.__timer as now:
|
| 585 |
+
self.expire(now)
|
| 586 |
+
LRUCache.__setitem__(self, key, value)
|
| 587 |
+
# Update or create expiry link
|
| 588 |
+
link = self.__links.get(key)
|
| 589 |
+
if link is not None:
|
| 590 |
+
link.unlink()
|
| 591 |
+
else:
|
| 592 |
+
link = _TTLLink()
|
| 593 |
+
self.__links[key] = link
|
| 594 |
+
link.key = key
|
| 595 |
+
link.expires = now + self.__ttl
|
| 596 |
+
# Append to end of expiry list (newest)
|
| 597 |
+
tail = self.__root.prev
|
| 598 |
+
link.prev = tail
|
| 599 |
+
link.next = self.__root
|
| 600 |
+
tail.next = link
|
| 601 |
+
self.__root.prev = link
|
| 602 |
+
|
| 603 |
+
def __delitem__(self, key: Any) -> None:
|
| 604 |
+
LRUCache.__delitem__(self, key)
|
| 605 |
+
link = self.__links.pop(key)
|
| 606 |
+
link.unlink()
|
| 607 |
+
|
| 608 |
+
def __iter__(self):
|
| 609 |
+
now = self.__timer()
|
| 610 |
+
for key in super().__iter__():
|
| 611 |
+
link = self.__links.get(key)
|
| 612 |
+
if link is not None and now < link.expires:
|
| 613 |
+
yield key
|
| 614 |
+
|
| 615 |
+
def __len__(self) -> int:
|
| 616 |
+
now = self.__timer()
|
| 617 |
+
# Single dict lookup via .get() instead of ``key in`` + ``[key]``.
|
| 618 |
+
links_get = self.__links.get
|
| 619 |
+
return sum(
|
| 620 |
+
1
|
| 621 |
+
for key in super().__iter__()
|
| 622 |
+
if (link := links_get(key)) is not None and now < link.expires
|
| 623 |
+
)
|
| 624 |
+
|
| 625 |
+
@property
|
| 626 |
+
def currsize(self) -> int | float:
|
| 627 |
+
"""Current size, excluding expired items."""
|
| 628 |
+
self.expire()
|
| 629 |
+
return super().currsize
|
| 630 |
+
|
| 631 |
+
def expire(self, time: float | None = None) -> list[tuple[Any, Any]]:
|
| 632 |
+
"""Remove expired items and return them as a list of ``(key, value)`` pairs."""
|
| 633 |
+
if time is None:
|
| 634 |
+
time = self.__timer()
|
| 635 |
+
root = self.__root
|
| 636 |
+
expired: list[tuple[Any, Any]] = []
|
| 637 |
+
link = root.next
|
| 638 |
+
while link is not root and link.expires <= time:
|
| 639 |
+
nxt = link.next
|
| 640 |
+
key = link.key
|
| 641 |
+
try:
|
| 642 |
+
value = Cache.__getitem__(self, key)
|
| 643 |
+
expired.append((key, value))
|
| 644 |
+
del self[key]
|
| 645 |
+
except KeyError:
|
| 646 |
+
pass
|
| 647 |
+
link = nxt
|
| 648 |
+
return expired
|
| 649 |
+
|
| 650 |
+
def clear(self) -> None:
|
| 651 |
+
LRUCache.clear(self)
|
| 652 |
+
self.__links.clear()
|
| 653 |
+
self.__root.prev = self.__root
|
| 654 |
+
self.__root.next = self.__root
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
# ── Decorators ─────────────────────────────────────────────────────────────
|
| 658 |
+
|
| 659 |
+
_MISS = object() # sentinel for cache lookup miss
|
| 660 |
+
|
| 661 |
+
|
| 662 |
+
class _Stats:
|
| 663 |
+
"""Mutable hit/miss counters shared by cache wrapper closures."""
|
| 664 |
+
|
| 665 |
+
__slots__ = ("hits", "misses")
|
| 666 |
+
|
| 667 |
+
def __init__(self) -> None:
|
| 668 |
+
self.hits = 0
|
| 669 |
+
self.misses = 0
|
| 670 |
+
|
| 671 |
+
|
| 672 |
+
def _get_cache_maxsize(
|
| 673 |
+
cache: collections.abc.MutableMapping,
|
| 674 |
+
) -> int | float | None:
|
| 675 |
+
"""Return maxsize if *cache* is a ``Cache`` instance, else ``None``."""
|
| 676 |
+
return cache.maxsize if isinstance(cache, Cache) else None
|
| 677 |
+
|
| 678 |
+
|
| 679 |
+
def _get_cache_currsize(cache: collections.abc.MutableMapping) -> int | float:
|
| 680 |
+
"""Return current size of *cache*."""
|
| 681 |
+
return cache.currsize if isinstance(cache, Cache) else len(cache)
|
| 682 |
+
|
| 683 |
+
|
| 684 |
+
def _attach_cache_info(
|
| 685 |
+
wrapper: Callable,
|
| 686 |
+
cache: collections.abc.MutableMapping,
|
| 687 |
+
lock: Any | None,
|
| 688 |
+
stats: _Stats,
|
| 689 |
+
maxsize: int | float | None,
|
| 690 |
+
) -> None:
|
| 691 |
+
"""Attach ``cache_info()`` and ``cache_clear()`` methods to *wrapper*."""
|
| 692 |
+
|
| 693 |
+
def cache_info() -> CacheInfo:
|
| 694 |
+
return CacheInfo(stats.hits, stats.misses, maxsize, _get_cache_currsize(cache))
|
| 695 |
+
|
| 696 |
+
def cache_clear() -> None:
|
| 697 |
+
with lock if lock else _no_lock():
|
| 698 |
+
cache.clear()
|
| 699 |
+
stats.hits = stats.misses = 0
|
| 700 |
+
|
| 701 |
+
wrapper.cache_info = cache_info # type: ignore[attr-defined]
|
| 702 |
+
wrapper.cache_clear = cache_clear # type: ignore[attr-defined]
|
| 703 |
+
|
| 704 |
+
|
| 705 |
+
def _attach_noop_info(
|
| 706 |
+
wrapper: Callable,
|
| 707 |
+
stats: _Stats,
|
| 708 |
+
) -> None:
|
| 709 |
+
"""Attach ``cache_info()`` and ``cache_clear()`` for a no-op wrapper."""
|
| 710 |
+
|
| 711 |
+
def cache_info() -> CacheInfo:
|
| 712 |
+
return CacheInfo(stats.hits, stats.misses, None, 0)
|
| 713 |
+
|
| 714 |
+
def cache_clear() -> None:
|
| 715 |
+
stats.hits = stats.misses = 0
|
| 716 |
+
|
| 717 |
+
wrapper.cache_info = cache_info # type: ignore[attr-defined]
|
| 718 |
+
wrapper.cache_clear = cache_clear # type: ignore[attr-defined]
|
| 719 |
+
|
| 720 |
+
|
| 721 |
+
def _make_sync_wrapper(
|
| 722 |
+
fn: Callable,
|
| 723 |
+
cache: collections.abc.MutableMapping | None,
|
| 724 |
+
key: Callable,
|
| 725 |
+
lock: Any | None,
|
| 726 |
+
info: bool,
|
| 727 |
+
) -> Callable:
|
| 728 |
+
"""Build a synchronous caching wrapper."""
|
| 729 |
+
stats = _Stats()
|
| 730 |
+
|
| 731 |
+
if cache is None:
|
| 732 |
+
|
| 733 |
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 734 |
+
stats.misses += 1
|
| 735 |
+
return fn(*args, **kwargs)
|
| 736 |
+
|
| 737 |
+
if info:
|
| 738 |
+
_attach_noop_info(wrapper, stats)
|
| 739 |
+
return wrapper
|
| 740 |
+
|
| 741 |
+
# Pre-cache method references to avoid attribute lookup per call.
|
| 742 |
+
cache_getitem = cache.__getitem__
|
| 743 |
+
cache_setitem = cache.__setitem__
|
| 744 |
+
cache_setdefault = cache.setdefault
|
| 745 |
+
|
| 746 |
+
if lock is not None:
|
| 747 |
+
|
| 748 |
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 749 |
+
k = key(*args, **kwargs)
|
| 750 |
+
with lock:
|
| 751 |
+
try:
|
| 752 |
+
result = cache_getitem(k)
|
| 753 |
+
except KeyError:
|
| 754 |
+
result = _MISS
|
| 755 |
+
if result is not _MISS:
|
| 756 |
+
stats.hits += 1
|
| 757 |
+
return result
|
| 758 |
+
stats.misses += 1
|
| 759 |
+
v = fn(*args, **kwargs)
|
| 760 |
+
# Inline thread-safe store (was _cache_store_default).
|
| 761 |
+
with lock:
|
| 762 |
+
try:
|
| 763 |
+
return cache_setdefault(k, v)
|
| 764 |
+
except ValueError:
|
| 765 |
+
return v
|
| 766 |
+
|
| 767 |
+
else:
|
| 768 |
+
|
| 769 |
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 770 |
+
k = key(*args, **kwargs)
|
| 771 |
+
try:
|
| 772 |
+
result = cache_getitem(k)
|
| 773 |
+
except KeyError:
|
| 774 |
+
stats.misses += 1
|
| 775 |
+
v = fn(*args, **kwargs)
|
| 776 |
+
try:
|
| 777 |
+
cache_setitem(k, v)
|
| 778 |
+
except ValueError:
|
| 779 |
+
pass
|
| 780 |
+
return v
|
| 781 |
+
stats.hits += 1
|
| 782 |
+
return result
|
| 783 |
+
|
| 784 |
+
if info:
|
| 785 |
+
_attach_cache_info(wrapper, cache, lock, stats, _get_cache_maxsize(cache))
|
| 786 |
+
|
| 787 |
+
return wrapper
|
| 788 |
+
|
| 789 |
+
|
| 790 |
+
def _make_async_wrapper(
|
| 791 |
+
fn: Callable,
|
| 792 |
+
cache: collections.abc.MutableMapping | None,
|
| 793 |
+
key: Callable,
|
| 794 |
+
lock: Any | None,
|
| 795 |
+
info: bool,
|
| 796 |
+
) -> Callable:
|
| 797 |
+
"""Build an asynchronous caching wrapper."""
|
| 798 |
+
stats = _Stats()
|
| 799 |
+
|
| 800 |
+
if cache is None:
|
| 801 |
+
|
| 802 |
+
async def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 803 |
+
stats.misses += 1
|
| 804 |
+
return await fn(*args, **kwargs)
|
| 805 |
+
|
| 806 |
+
if info:
|
| 807 |
+
_attach_noop_info(wrapper, stats)
|
| 808 |
+
return wrapper
|
| 809 |
+
|
| 810 |
+
# Pre-cache method references to avoid attribute lookup per call.
|
| 811 |
+
cache_getitem = cache.__getitem__
|
| 812 |
+
cache_setitem = cache.__setitem__
|
| 813 |
+
cache_setdefault = cache.setdefault
|
| 814 |
+
|
| 815 |
+
if lock is not None:
|
| 816 |
+
|
| 817 |
+
async def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 818 |
+
k = key(*args, **kwargs)
|
| 819 |
+
async with lock:
|
| 820 |
+
try:
|
| 821 |
+
result = cache_getitem(k)
|
| 822 |
+
except KeyError:
|
| 823 |
+
result = _MISS
|
| 824 |
+
if result is not _MISS:
|
| 825 |
+
stats.hits += 1
|
| 826 |
+
return result
|
| 827 |
+
stats.misses += 1
|
| 828 |
+
v = await fn(*args, **kwargs)
|
| 829 |
+
# Inline thread-safe store (was _cache_store_default).
|
| 830 |
+
async with lock:
|
| 831 |
+
try:
|
| 832 |
+
return cache_setdefault(k, v)
|
| 833 |
+
except ValueError:
|
| 834 |
+
return v
|
| 835 |
+
|
| 836 |
+
else:
|
| 837 |
+
|
| 838 |
+
async def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 839 |
+
k = key(*args, **kwargs)
|
| 840 |
+
try:
|
| 841 |
+
result = cache_getitem(k)
|
| 842 |
+
except KeyError:
|
| 843 |
+
stats.misses += 1
|
| 844 |
+
v = await fn(*args, **kwargs)
|
| 845 |
+
try:
|
| 846 |
+
cache_setitem(k, v)
|
| 847 |
+
except ValueError:
|
| 848 |
+
pass
|
| 849 |
+
return v
|
| 850 |
+
stats.hits += 1
|
| 851 |
+
return result
|
| 852 |
+
|
| 853 |
+
if info:
|
| 854 |
+
_attach_cache_info(wrapper, cache, lock, stats, _get_cache_maxsize(cache))
|
| 855 |
+
|
| 856 |
+
return wrapper
|
| 857 |
+
|
| 858 |
+
|
| 859 |
+
class _no_lock:
|
| 860 |
+
"""Dummy context manager when no lock is provided."""
|
| 861 |
+
|
| 862 |
+
def __enter__(self) -> None:
|
| 863 |
+
pass
|
| 864 |
+
|
| 865 |
+
def __exit__(self, *exc: Any) -> None:
|
| 866 |
+
pass
|
| 867 |
+
|
| 868 |
+
|
| 869 |
+
def cached(
|
| 870 |
+
cache: Cache | collections.abc.MutableMapping | None,
|
| 871 |
+
*,
|
| 872 |
+
key: Callable[..., Any] = hashkey,
|
| 873 |
+
lock: Any | None = None,
|
| 874 |
+
info: bool = False,
|
| 875 |
+
) -> Callable:
|
| 876 |
+
"""Decorator to wrap a function with a memoizing callable.
|
| 877 |
+
|
| 878 |
+
Args:
|
| 879 |
+
cache: A cache instance (any MutableMapping) or ``None`` to disable.
|
| 880 |
+
key: Key function (default ``hashkey``).
|
| 881 |
+
lock: A lock object (``threading.Lock`` for sync, ``asyncio.Lock``
|
| 882 |
+
for async). The lock is released while the wrapped function
|
| 883 |
+
executes.
|
| 884 |
+
info: If ``True``, attach ``cache_info()`` and ``cache_clear()``
|
| 885 |
+
methods to the wrapper.
|
| 886 |
+
"""
|
| 887 |
+
|
| 888 |
+
def decorator(fn: Callable) -> Callable:
|
| 889 |
+
if inspect.iscoroutinefunction(fn):
|
| 890 |
+
wrapper = _make_async_wrapper(fn, cache, key, lock, info)
|
| 891 |
+
else:
|
| 892 |
+
wrapper = _make_sync_wrapper(fn, cache, key, lock, info)
|
| 893 |
+
wrapper.cache = cache # type: ignore[attr-defined]
|
| 894 |
+
wrapper.cache_key = key # type: ignore[attr-defined]
|
| 895 |
+
wrapper.cache_lock = lock # type: ignore[attr-defined]
|
| 896 |
+
return functools.update_wrapper(wrapper, fn)
|
| 897 |
+
|
| 898 |
+
return decorator
|
| 899 |
+
|
| 900 |
+
|
| 901 |
+
# ── Convenience Decorators ─────────────────────────────────────────────────
|
| 902 |
+
|
| 903 |
+
|
| 904 |
+
def lru_cache(
|
| 905 |
+
fn: Callable[..., Any] | None = None,
|
| 906 |
+
*,
|
| 907 |
+
maxsize: int = 128,
|
| 908 |
+
key: Callable[..., Any] = hashkey,
|
| 909 |
+
lock: Any | None = None,
|
| 910 |
+
info: bool = True,
|
| 911 |
+
) -> Callable[..., Any]:
|
| 912 |
+
"""LRU cache decorator. Auto-detects sync/async.
|
| 913 |
+
|
| 914 |
+
Unlike ``functools.lru_cache``, supports TTL (via ``ttl_cache``),
|
| 915 |
+
async functions, and explicit lock objects.
|
| 916 |
+
|
| 917 |
+
Args:
|
| 918 |
+
maxsize: Maximum number of cached results.
|
| 919 |
+
key: Key function (default ``hashkey``).
|
| 920 |
+
lock: Optional lock for thread/async safety.
|
| 921 |
+
info: Attach ``cache_info()``/``cache_clear()`` (default ``True``).
|
| 922 |
+
"""
|
| 923 |
+
|
| 924 |
+
def decorator(fn: Callable) -> Callable:
|
| 925 |
+
c = LRUCache(maxsize)
|
| 926 |
+
return cached(c, key=key, lock=lock, info=info)(fn)
|
| 927 |
+
|
| 928 |
+
if fn is not None:
|
| 929 |
+
return decorator(fn)
|
| 930 |
+
return decorator
|
| 931 |
+
|
| 932 |
+
|
| 933 |
+
def fifo_cache(
|
| 934 |
+
fn: Callable[..., Any] | None = None,
|
| 935 |
+
*,
|
| 936 |
+
maxsize: int = 128,
|
| 937 |
+
key: Callable[..., Any] = hashkey,
|
| 938 |
+
lock: Any | None = None,
|
| 939 |
+
info: bool = True,
|
| 940 |
+
) -> Callable[..., Any]:
|
| 941 |
+
"""FIFO cache decorator. Auto-detects sync/async.
|
| 942 |
+
|
| 943 |
+
Evicts the oldest inserted item regardless of access pattern.
|
| 944 |
+
"""
|
| 945 |
+
|
| 946 |
+
def decorator(fn: Callable) -> Callable:
|
| 947 |
+
c = FIFOCache(maxsize)
|
| 948 |
+
return cached(c, key=key, lock=lock, info=info)(fn)
|
| 949 |
+
|
| 950 |
+
if fn is not None:
|
| 951 |
+
return decorator(fn)
|
| 952 |
+
return decorator
|
| 953 |
+
|
| 954 |
+
|
| 955 |
+
def lfu_cache(
|
| 956 |
+
fn: Callable[..., Any] | None = None,
|
| 957 |
+
*,
|
| 958 |
+
maxsize: int = 128,
|
| 959 |
+
key: Callable[..., Any] = hashkey,
|
| 960 |
+
lock: Any | None = None,
|
| 961 |
+
info: bool = True,
|
| 962 |
+
) -> Callable[..., Any]:
|
| 963 |
+
"""LFU cache decorator. Auto-detects sync/async.
|
| 964 |
+
|
| 965 |
+
Evicts the least frequently accessed item.
|
| 966 |
+
"""
|
| 967 |
+
|
| 968 |
+
def decorator(fn: Callable) -> Callable:
|
| 969 |
+
c = LFUCache(maxsize)
|
| 970 |
+
return cached(c, key=key, lock=lock, info=info)(fn)
|
| 971 |
+
|
| 972 |
+
if fn is not None:
|
| 973 |
+
return decorator(fn)
|
| 974 |
+
return decorator
|
| 975 |
+
|
| 976 |
+
|
| 977 |
+
def ttl_cache(
|
| 978 |
+
fn: Callable[..., Any] | None = None,
|
| 979 |
+
*,
|
| 980 |
+
maxsize: int = 128,
|
| 981 |
+
ttl: float = 600,
|
| 982 |
+
timer: Callable[[], float] = time.monotonic,
|
| 983 |
+
key: Callable[..., Any] = hashkey,
|
| 984 |
+
lock: Any | None = None,
|
| 985 |
+
info: bool = True,
|
| 986 |
+
) -> Callable[..., Any]:
|
| 987 |
+
"""TTL cache decorator. Auto-detects sync/async.
|
| 988 |
+
|
| 989 |
+
Items expire after *ttl* seconds (default 600 = 10 minutes).
|
| 990 |
+
Eviction follows LRU order when the cache is full.
|
| 991 |
+
"""
|
| 992 |
+
|
| 993 |
+
def decorator(fn: Callable) -> Callable:
|
| 994 |
+
c = TTLCache(maxsize, ttl, timer=timer)
|
| 995 |
+
return cached(c, key=key, lock=lock, info=info)(fn)
|
| 996 |
+
|
| 997 |
+
if fn is not None:
|
| 998 |
+
return decorator(fn)
|
| 999 |
+
return decorator
|
| 1000 |
+
|
| 1001 |
+
|
| 1002 |
+
# ── Public API ─────────────────────────────────────────────────────────────
|
| 1003 |
+
|
| 1004 |
+
__all__ = [
|
| 1005 |
+
# Cache classes
|
| 1006 |
+
"Cache",
|
| 1007 |
+
"FIFOCache",
|
| 1008 |
+
"LFUCache",
|
| 1009 |
+
"LRUCache",
|
| 1010 |
+
"TTLCache",
|
| 1011 |
+
# Decorators
|
| 1012 |
+
"cached",
|
| 1013 |
+
"fifo_cache",
|
| 1014 |
+
"lfu_cache",
|
| 1015 |
+
"lru_cache",
|
| 1016 |
+
"ttl_cache",
|
| 1017 |
+
# Key functions
|
| 1018 |
+
"hashkey",
|
| 1019 |
+
"methodkey",
|
| 1020 |
+
"typedkey",
|
| 1021 |
+
# Stats
|
| 1022 |
+
"CacheInfo",
|
| 1023 |
+
]
|
|
@@ -0,0 +1,713 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.3.0"
|
| 3 |
+
# deps = ["dotenv", "yaml", "jsonc"]
|
| 4 |
+
# tier = "subsystem"
|
| 5 |
+
# category = "config"
|
| 6 |
+
# note = "Install/update via `zerodep add config`"
|
| 7 |
+
# ///
|
| 8 |
+
|
| 9 |
+
"""Unified configuration loader — zero dependencies, stdlib only, Python 3.10+.
|
| 10 |
+
|
| 11 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 12 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 13 |
+
|
| 14 |
+
Drop-in replacement for python-decouple / dynaconf core functionality.
|
| 15 |
+
Loads settings from environment variables, .env files, and config files
|
| 16 |
+
(JSON, JSONC, YAML, TOML, INI) with type coercion and prefix support.
|
| 17 |
+
|
| 18 |
+
Example::
|
| 19 |
+
|
| 20 |
+
from config import config, setup
|
| 21 |
+
|
| 22 |
+
# Auto-discovers .env, reads env vars
|
| 23 |
+
setup(prefix="MYAPP_")
|
| 24 |
+
debug = config("DEBUG", default=False, cast=bool)
|
| 25 |
+
port = config("PORT", default=8000, cast=int)
|
| 26 |
+
hosts = config("ALLOWED_HOSTS", cast=Csv())
|
| 27 |
+
|
| 28 |
+
# Or use Config directly
|
| 29 |
+
cfg = Config(config_path="settings.yaml", prefix="MYAPP_")
|
| 30 |
+
db_host = cfg("DATABASE__HOST", default="localhost")
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
from __future__ import annotations
|
| 34 |
+
|
| 35 |
+
import configparser
|
| 36 |
+
import json
|
| 37 |
+
import os
|
| 38 |
+
import sys
|
| 39 |
+
from pathlib import Path
|
| 40 |
+
from typing import Any, Callable, Sequence
|
| 41 |
+
|
| 42 |
+
__all__ = [
|
| 43 |
+
# Sentinels
|
| 44 |
+
"MISSING",
|
| 45 |
+
# Exceptions
|
| 46 |
+
"ConfigError",
|
| 47 |
+
"UndefinedValueError",
|
| 48 |
+
# Type coercion helpers
|
| 49 |
+
"Csv",
|
| 50 |
+
"Choices",
|
| 51 |
+
# Main class
|
| 52 |
+
"Config",
|
| 53 |
+
# Module-level convenience
|
| 54 |
+
"setup",
|
| 55 |
+
"config",
|
| 56 |
+
]
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _ensure_sibling_path(name: str) -> str:
|
| 60 |
+
"""Return the sibling module directory and prepend it to ``sys.path``."""
|
| 61 |
+
sibling_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", name)
|
| 62 |
+
if sibling_dir not in sys.path:
|
| 63 |
+
sys.path.insert(0, sibling_dir)
|
| 64 |
+
return sibling_dir
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
# ── Sibling module loaders (lazy) ───────────────────────────────────────────
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _load_dotenv_helpers() -> tuple[Callable[..., Any], Callable[..., Any]]:
|
| 71 |
+
"""Load sibling ``dotenv`` helpers on demand."""
|
| 72 |
+
_ensure_sibling_path("dotenv")
|
| 73 |
+
sys.modules.pop("dotenv", None)
|
| 74 |
+
try:
|
| 75 |
+
from dotenv import dotenv_values, find_dotenv
|
| 76 |
+
except ImportError as exc:
|
| 77 |
+
raise ImportError(
|
| 78 |
+
".env loading requires the sibling dotenv module. "
|
| 79 |
+
"Copy dotenv/dotenv.py into your project, "
|
| 80 |
+
"or set dotenv_path=None to disable."
|
| 81 |
+
) from exc
|
| 82 |
+
return dotenv_values, find_dotenv
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _load_yaml_loader() -> Callable[[str], Any]:
|
| 86 |
+
"""Load the sibling ``yaml`` module's ``load`` function on demand."""
|
| 87 |
+
_ensure_sibling_path("yaml")
|
| 88 |
+
sys.modules.pop("yaml", None)
|
| 89 |
+
try:
|
| 90 |
+
from yaml import load as yaml_load
|
| 91 |
+
except ImportError as exc:
|
| 92 |
+
raise ImportError(
|
| 93 |
+
"YAML config files require the sibling yaml module. "
|
| 94 |
+
"Copy yaml/yaml.py into your project."
|
| 95 |
+
) from exc
|
| 96 |
+
return yaml_load
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _load_jsonc_loader() -> Callable[[str], Any] | None:
|
| 100 |
+
"""Load the sibling ``jsonc`` module's ``loads`` function if available."""
|
| 101 |
+
_ensure_sibling_path("jsonc")
|
| 102 |
+
sys.modules.pop("jsonc", None)
|
| 103 |
+
try:
|
| 104 |
+
from jsonc import loads as jsonc_loads
|
| 105 |
+
except ImportError:
|
| 106 |
+
return None
|
| 107 |
+
return jsonc_loads
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
# ── Stdlib tomllib (Python 3.11+) ───────────────────────────────────────────
|
| 111 |
+
|
| 112 |
+
try:
|
| 113 |
+
import tomllib # type: ignore[import-not-found]
|
| 114 |
+
|
| 115 |
+
_HAS_TOML = True
|
| 116 |
+
except ModuleNotFoundError:
|
| 117 |
+
_HAS_TOML = False
|
| 118 |
+
|
| 119 |
+
# ── Sentinels ───────────────────────────────────────────────────────────────
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
class _Missing:
|
| 123 |
+
"""Sentinel for missing configuration values."""
|
| 124 |
+
|
| 125 |
+
_instance: _Missing | None = None
|
| 126 |
+
|
| 127 |
+
def __new__(cls) -> _Missing:
|
| 128 |
+
if cls._instance is None:
|
| 129 |
+
cls._instance = super().__new__(cls)
|
| 130 |
+
return cls._instance
|
| 131 |
+
|
| 132 |
+
def __repr__(self) -> str:
|
| 133 |
+
return "MISSING"
|
| 134 |
+
|
| 135 |
+
def __bool__(self) -> bool:
|
| 136 |
+
return False
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
MISSING = _Missing()
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class _Auto:
|
| 143 |
+
"""Sentinel indicating auto-discovery of .env files."""
|
| 144 |
+
|
| 145 |
+
_instance: _Auto | None = None
|
| 146 |
+
|
| 147 |
+
def __new__(cls) -> _Auto:
|
| 148 |
+
if cls._instance is None:
|
| 149 |
+
cls._instance = super().__new__(cls)
|
| 150 |
+
return cls._instance
|
| 151 |
+
|
| 152 |
+
def __repr__(self) -> str:
|
| 153 |
+
return "AUTO"
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
_AUTO = _Auto()
|
| 157 |
+
|
| 158 |
+
# ── Sentinel for injection parameters ──────────────────────────────────────
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
class _Unset:
|
| 162 |
+
"""Sentinel indicating 'use default sibling auto-discovery'."""
|
| 163 |
+
|
| 164 |
+
_instance: _Unset | None = None
|
| 165 |
+
|
| 166 |
+
def __new__(cls) -> _Unset:
|
| 167 |
+
if cls._instance is None:
|
| 168 |
+
cls._instance = super().__new__(cls)
|
| 169 |
+
return cls._instance
|
| 170 |
+
|
| 171 |
+
def __repr__(self) -> str:
|
| 172 |
+
return "UNSET"
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
_UNSET = _Unset()
|
| 176 |
+
|
| 177 |
+
# ── Exceptions ──────────────────────────────────────────────────────────────
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
class ConfigError(Exception):
|
| 181 |
+
"""Base exception for all config operations."""
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
class UndefinedValueError(ConfigError):
|
| 185 |
+
"""Raised when a required configuration value is missing."""
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
# ── Bool constants ──────────────────────────────────────────────────────────
|
| 189 |
+
|
| 190 |
+
_TRUTHY = frozenset({"1", "true", "yes", "on", "t", "y"})
|
| 191 |
+
_FALSY = frozenset({"0", "false", "no", "off", "f", "n", ""})
|
| 192 |
+
|
| 193 |
+
# ── Type coercion helpers ───────────────────────────────────────────────────
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
class Csv:
|
| 197 |
+
"""Parse comma-separated values with optional per-item casting.
|
| 198 |
+
|
| 199 |
+
Example::
|
| 200 |
+
|
| 201 |
+
Csv()("a, b, c") # ["a", "b", "c"]
|
| 202 |
+
Csv(cast=int)("1,2,3") # [1, 2, 3]
|
| 203 |
+
Csv(delimiter=";")("a;b") # ["a", "b"]
|
| 204 |
+
|
| 205 |
+
Args:
|
| 206 |
+
cast: Callable applied to each item after splitting.
|
| 207 |
+
delimiter: String to split on.
|
| 208 |
+
strip: Characters to strip from each item. Use ``" %s"`` to strip
|
| 209 |
+
whitespace around the format-string placeholder (default).
|
| 210 |
+
post_process: Callable applied to the final list (e.g. ``tuple``).
|
| 211 |
+
"""
|
| 212 |
+
|
| 213 |
+
def __init__(
|
| 214 |
+
self,
|
| 215 |
+
cast: Callable[[str], Any] = str,
|
| 216 |
+
delimiter: str = ",",
|
| 217 |
+
strip: str = " %s",
|
| 218 |
+
post_process: Callable[[list[Any]], Any] = list,
|
| 219 |
+
) -> None:
|
| 220 |
+
self.cast = cast
|
| 221 |
+
self.delimiter = delimiter
|
| 222 |
+
self.strip = strip
|
| 223 |
+
self.post_process = post_process
|
| 224 |
+
|
| 225 |
+
def __call__(self, value: str) -> Any:
|
| 226 |
+
if isinstance(value, (list, tuple)):
|
| 227 |
+
return self.post_process([self.cast(item) for item in value])
|
| 228 |
+
parts = str(value).split(self.delimiter)
|
| 229 |
+
result = []
|
| 230 |
+
for part in parts:
|
| 231 |
+
item = part.strip(self.strip.replace("%s", "")) if self.strip else part
|
| 232 |
+
if item or not self.strip:
|
| 233 |
+
result.append(self.cast(item))
|
| 234 |
+
return self.post_process(result)
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
class Choices:
|
| 238 |
+
"""Validate that a value belongs to a fixed set.
|
| 239 |
+
|
| 240 |
+
Example::
|
| 241 |
+
|
| 242 |
+
Choices(["dev", "staging", "prod"])("dev") # "dev"
|
| 243 |
+
Choices([1, 2, 3], cast=int)("2") # 2
|
| 244 |
+
|
| 245 |
+
Args:
|
| 246 |
+
choices: Allowed values (after casting).
|
| 247 |
+
cast: Callable applied before validation.
|
| 248 |
+
"""
|
| 249 |
+
|
| 250 |
+
def __init__(
|
| 251 |
+
self,
|
| 252 |
+
choices: Sequence[Any],
|
| 253 |
+
cast: Callable[[str], Any] = str,
|
| 254 |
+
) -> None:
|
| 255 |
+
self.choices = choices
|
| 256 |
+
self.cast = cast
|
| 257 |
+
|
| 258 |
+
def __call__(self, value: str) -> Any:
|
| 259 |
+
casted = self.cast(value)
|
| 260 |
+
if casted not in self.choices:
|
| 261 |
+
raise ValueError(
|
| 262 |
+
f"{casted!r} is not a valid choice. Allowed: {list(self.choices)}"
|
| 263 |
+
)
|
| 264 |
+
return casted
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
# ── File loaders ────────────────────────────────────────────────────────────
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
def _load_json_file(path: str | os.PathLike[str]) -> dict[str, Any]:
|
| 271 |
+
"""Load a JSON file and return a dict."""
|
| 272 |
+
with open(path, encoding="utf-8") as f:
|
| 273 |
+
data = json.load(f)
|
| 274 |
+
if not isinstance(data, dict):
|
| 275 |
+
raise ValueError(f"JSON config must be an object, got {type(data).__name__}")
|
| 276 |
+
return data
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def _load_jsonc_file(path: str | os.PathLike[str]) -> dict[str, Any]:
|
| 280 |
+
"""Load a JSONC file, falling back to plain JSON if jsonc module is unavailable."""
|
| 281 |
+
with open(path, encoding="utf-8") as f:
|
| 282 |
+
text = f.read()
|
| 283 |
+
jsonc_loads = _load_jsonc_loader()
|
| 284 |
+
if jsonc_loads is not None:
|
| 285 |
+
data = jsonc_loads(text)
|
| 286 |
+
else:
|
| 287 |
+
data = json.loads(text)
|
| 288 |
+
if not isinstance(data, dict):
|
| 289 |
+
raise ValueError(f"JSONC config must be an object, got {type(data).__name__}")
|
| 290 |
+
return data
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def _load_yaml_file(path: str | os.PathLike[str]) -> dict[str, Any]:
|
| 294 |
+
"""Load a YAML file using the sibling yaml module."""
|
| 295 |
+
yaml_load = _load_yaml_loader()
|
| 296 |
+
with open(path, encoding="utf-8") as f:
|
| 297 |
+
text = f.read()
|
| 298 |
+
data = yaml_load(text)
|
| 299 |
+
if data is None:
|
| 300 |
+
return {}
|
| 301 |
+
if not isinstance(data, dict):
|
| 302 |
+
raise ValueError(f"YAML config must be a mapping, got {type(data).__name__}")
|
| 303 |
+
return data
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def _load_toml_file(path: str | os.PathLike[str]) -> dict[str, Any]:
|
| 307 |
+
"""Load a TOML file using stdlib tomllib (Python 3.11+)."""
|
| 308 |
+
if not _HAS_TOML:
|
| 309 |
+
raise ImportError("TOML config files require Python 3.11+ (tomllib).")
|
| 310 |
+
with open(path, "rb") as f:
|
| 311 |
+
return tomllib.load(f)
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def _load_ini_file(
|
| 315 |
+
path: str | os.PathLike[str], separator: str = "__"
|
| 316 |
+
) -> dict[str, Any]:
|
| 317 |
+
"""Load an INI/CFG file and flatten sections into separator-joined keys.
|
| 318 |
+
|
| 319 |
+
Example: ``[database]`` section with ``host = localhost`` becomes
|
| 320 |
+
``database__host = localhost`` (using the default separator).
|
| 321 |
+
"""
|
| 322 |
+
parser = configparser.ConfigParser()
|
| 323 |
+
parser.read(path, encoding="utf-8")
|
| 324 |
+
data: dict[str, Any] = {}
|
| 325 |
+
for section in parser.sections():
|
| 326 |
+
for key, value in parser.items(section):
|
| 327 |
+
flat_key = f"{section}{separator}{key}"
|
| 328 |
+
data[flat_key] = value
|
| 329 |
+
if parser.defaults():
|
| 330 |
+
for key, value in parser.defaults().items():
|
| 331 |
+
data[key] = value
|
| 332 |
+
return data
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
_LOADERS: dict[str, Callable[..., dict[str, Any]]] = {
|
| 336 |
+
".json": _load_json_file,
|
| 337 |
+
".jsonc": _load_jsonc_file,
|
| 338 |
+
".yaml": _load_yaml_file,
|
| 339 |
+
".yml": _load_yaml_file,
|
| 340 |
+
".toml": _load_toml_file,
|
| 341 |
+
".ini": _load_ini_file,
|
| 342 |
+
".cfg": _load_ini_file,
|
| 343 |
+
}
|
| 344 |
+
|
| 345 |
+
# ── Internal helpers ────────────────────────────────────────────────────────
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
def _cast_bool(value: Any) -> bool:
|
| 349 |
+
"""Convert a value to bool using common truthy/falsy strings."""
|
| 350 |
+
if isinstance(value, bool):
|
| 351 |
+
return value
|
| 352 |
+
s = str(value).lower().strip()
|
| 353 |
+
if s in _TRUTHY:
|
| 354 |
+
return True
|
| 355 |
+
if s in _FALSY:
|
| 356 |
+
return False
|
| 357 |
+
raise ValueError(
|
| 358 |
+
f"Cannot convert {value!r} to bool. Use one of: {sorted(_TRUTHY | _FALSY)}"
|
| 359 |
+
)
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def _cast_list(value: Any) -> list[Any]:
|
| 363 |
+
"""Convert a value to list: try JSON array first, then comma-split."""
|
| 364 |
+
if isinstance(value, list):
|
| 365 |
+
return value
|
| 366 |
+
if isinstance(value, tuple):
|
| 367 |
+
return list(value)
|
| 368 |
+
s = str(value).strip()
|
| 369 |
+
if s.startswith("["):
|
| 370 |
+
try:
|
| 371 |
+
result = json.loads(s)
|
| 372 |
+
if isinstance(result, list):
|
| 373 |
+
return result
|
| 374 |
+
except (json.JSONDecodeError, ValueError):
|
| 375 |
+
pass
|
| 376 |
+
return [item.strip() for item in s.split(",") if item.strip()]
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def _cast_tuple(value: Any) -> tuple[Any, ...]:
|
| 380 |
+
"""Convert a value to tuple via _cast_list."""
|
| 381 |
+
return tuple(_cast_list(value))
|
| 382 |
+
|
| 383 |
+
|
| 384 |
+
def _apply_cast(value: Any, cast: type | Callable[..., Any] | None) -> Any:
|
| 385 |
+
"""Apply a type cast to a value."""
|
| 386 |
+
if cast is None:
|
| 387 |
+
return value
|
| 388 |
+
if cast is bool:
|
| 389 |
+
return _cast_bool(value)
|
| 390 |
+
if cast is list:
|
| 391 |
+
return _cast_list(value)
|
| 392 |
+
if cast is tuple:
|
| 393 |
+
return _cast_tuple(value)
|
| 394 |
+
return cast(value)
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
def _deep_get(data: dict[str, Any], parts: list[str]) -> Any:
|
| 398 |
+
"""Retrieve a nested value from a dict using a list of key parts.
|
| 399 |
+
|
| 400 |
+
Returns *MISSING* if any intermediate key is missing or not a dict.
|
| 401 |
+
"""
|
| 402 |
+
current: Any = data
|
| 403 |
+
for part in parts:
|
| 404 |
+
if not isinstance(current, dict):
|
| 405 |
+
return MISSING
|
| 406 |
+
# Try exact key first, then case-insensitive
|
| 407 |
+
if part in current:
|
| 408 |
+
current = current[part]
|
| 409 |
+
else:
|
| 410 |
+
lower = part.lower()
|
| 411 |
+
found = False
|
| 412 |
+
for k in current:
|
| 413 |
+
if k.lower() == lower:
|
| 414 |
+
current = current[k]
|
| 415 |
+
found = True
|
| 416 |
+
break
|
| 417 |
+
if not found:
|
| 418 |
+
return MISSING
|
| 419 |
+
return current
|
| 420 |
+
|
| 421 |
+
|
| 422 |
+
def _flatten_dict(
|
| 423 |
+
data: dict[str, Any], separator: str = "__", prefix: str = ""
|
| 424 |
+
) -> dict[str, Any]:
|
| 425 |
+
"""Flatten a nested dict into separator-joined keys."""
|
| 426 |
+
result: dict[str, Any] = {}
|
| 427 |
+
for key, value in data.items():
|
| 428 |
+
full_key = f"{prefix}{separator}{key}" if prefix else key
|
| 429 |
+
if isinstance(value, dict):
|
| 430 |
+
result.update(_flatten_dict(value, separator, full_key))
|
| 431 |
+
else:
|
| 432 |
+
result[full_key] = value
|
| 433 |
+
return result
|
| 434 |
+
|
| 435 |
+
|
| 436 |
+
# ── Main Config class ───────────────────────────────────────────────────────
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
class Config:
|
| 440 |
+
"""Unified configuration loader with multi-source support.
|
| 441 |
+
|
| 442 |
+
Sources are checked in priority order (highest first):
|
| 443 |
+
|
| 444 |
+
1. Environment variables (``os.environ``), optionally prefixed
|
| 445 |
+
2. ``.env`` file values (via sibling dotenv module)
|
| 446 |
+
3. Config file values (JSON, JSONC, YAML, TOML, INI)
|
| 447 |
+
4. Default value passed to ``get()``/``__call__()``
|
| 448 |
+
|
| 449 |
+
Args:
|
| 450 |
+
dotenv_path: Path to ``.env`` file. Use ``None`` to disable,
|
| 451 |
+
or omit (default ``_AUTO``) to auto-discover by searching
|
| 452 |
+
upward from the current directory.
|
| 453 |
+
config_path: Path to a config file. Format is detected from the
|
| 454 |
+
file extension.
|
| 455 |
+
prefix: Prefix for environment variable lookups. For example,
|
| 456 |
+
``prefix="MYAPP_"`` means looking up ``"PORT"`` checks
|
| 457 |
+
``os.environ["MYAPP_PORT"]``.
|
| 458 |
+
separator: Separator for nested key access. Default ``"__"``
|
| 459 |
+
means ``"DATABASE__HOST"`` resolves to ``data["DATABASE"]["HOST"]``
|
| 460 |
+
or ``data["database"]["host"]`` in config files.
|
| 461 |
+
loaders: Override the file-format loader registry. Defaults to
|
| 462 |
+
``_UNSET`` (use built-in loaders with sibling auto-discovery
|
| 463 |
+
for yaml/jsonc). Pass a ``dict`` mapping extensions (e.g.
|
| 464 |
+
``".yaml"``) to loader callables to replace the defaults.
|
| 465 |
+
Stdlib loaders (json, toml, ini) are always available unless
|
| 466 |
+
explicitly overridden.
|
| 467 |
+
dotenv_loader: Override the dotenv loading mechanism. Defaults to
|
| 468 |
+
``_UNSET`` (auto-discover sibling ``dotenv`` module). Pass
|
| 469 |
+
``None`` to disable .env loading entirely, or a callable that
|
| 470 |
+
returns ``(dotenv_values_fn, find_dotenv_fn)`` to inject a
|
| 471 |
+
custom implementation.
|
| 472 |
+
|
| 473 |
+
Example::
|
| 474 |
+
|
| 475 |
+
cfg = Config(config_path="settings.yaml", prefix="MYAPP_")
|
| 476 |
+
debug = cfg("DEBUG", default=False, cast=bool)
|
| 477 |
+
db_host = cfg("DATABASE__HOST", default="localhost")
|
| 478 |
+
"""
|
| 479 |
+
|
| 480 |
+
def __init__(
|
| 481 |
+
self,
|
| 482 |
+
*,
|
| 483 |
+
dotenv_path: str | os.PathLike[str] | None | _Auto = _AUTO,
|
| 484 |
+
config_path: str | os.PathLike[str] | None = None,
|
| 485 |
+
prefix: str = "",
|
| 486 |
+
separator: str = "__",
|
| 487 |
+
loaders: dict[str, Callable[..., dict[str, Any]]] | None | _Unset = _UNSET,
|
| 488 |
+
dotenv_loader: Callable[[], tuple[Callable[..., Any], Callable[..., Any]]]
|
| 489 |
+
| None
|
| 490 |
+
| _Unset = _UNSET,
|
| 491 |
+
) -> None:
|
| 492 |
+
self._prefix = prefix
|
| 493 |
+
self._separator = separator
|
| 494 |
+
self._dotenv_data: dict[str, str | None] = {}
|
| 495 |
+
self._config_data: dict[str, Any] = {}
|
| 496 |
+
self._loaders: dict[str, Callable[..., dict[str, Any]]]
|
| 497 |
+
|
| 498 |
+
# Resolve loader registry
|
| 499 |
+
if isinstance(loaders, _Unset):
|
| 500 |
+
self._loaders = _LOADERS
|
| 501 |
+
elif loaders is None:
|
| 502 |
+
self._loaders = {}
|
| 503 |
+
else:
|
| 504 |
+
self._loaders = loaders
|
| 505 |
+
|
| 506 |
+
# Load .env file
|
| 507 |
+
if dotenv_loader is None:
|
| 508 |
+
pass # .env explicitly disabled
|
| 509 |
+
elif isinstance(dotenv_path, _Auto):
|
| 510 |
+
try:
|
| 511 |
+
if isinstance(dotenv_loader, _Unset):
|
| 512 |
+
dotenv_values, find_dotenv = _load_dotenv_helpers()
|
| 513 |
+
else:
|
| 514 |
+
dotenv_values, find_dotenv = dotenv_loader()
|
| 515 |
+
except ImportError:
|
| 516 |
+
pass
|
| 517 |
+
else:
|
| 518 |
+
found = find_dotenv(usecwd=True)
|
| 519 |
+
if found:
|
| 520 |
+
self._dotenv_data = dotenv_values(found)
|
| 521 |
+
elif dotenv_path is not None:
|
| 522 |
+
if isinstance(dotenv_loader, _Unset):
|
| 523 |
+
dotenv_values, _find_dotenv = _load_dotenv_helpers()
|
| 524 |
+
else:
|
| 525 |
+
dotenv_values, _find_dotenv = dotenv_loader()
|
| 526 |
+
self._dotenv_data = dotenv_values(str(dotenv_path))
|
| 527 |
+
|
| 528 |
+
# Load config file
|
| 529 |
+
if config_path is not None:
|
| 530 |
+
self._load_config_file(config_path)
|
| 531 |
+
|
| 532 |
+
def _load_config_file(self, path: str | os.PathLike[str]) -> None:
|
| 533 |
+
"""Load a config file based on its extension."""
|
| 534 |
+
p = Path(path)
|
| 535 |
+
ext = p.suffix.lower()
|
| 536 |
+
loader = self._loaders.get(ext)
|
| 537 |
+
if loader is None:
|
| 538 |
+
raise ValueError(
|
| 539 |
+
f"Unsupported config file format: {ext!r}. "
|
| 540 |
+
f"Supported: {', '.join(sorted(self._loaders))}"
|
| 541 |
+
)
|
| 542 |
+
if loader in (_load_ini_file,):
|
| 543 |
+
self._config_data = loader(p, separator=self._separator)
|
| 544 |
+
else:
|
| 545 |
+
self._config_data = loader(p)
|
| 546 |
+
|
| 547 |
+
def _lookup(self, key: str) -> Any:
|
| 548 |
+
"""Look up a key across all sources in priority order.
|
| 549 |
+
|
| 550 |
+
Returns the value if found, or *MISSING* if not found anywhere.
|
| 551 |
+
"""
|
| 552 |
+
# 1. Environment variables (with prefix)
|
| 553 |
+
env_key = f"{self._prefix}{key}" if self._prefix else key
|
| 554 |
+
env_val = os.environ.get(env_key)
|
| 555 |
+
if env_val is not None:
|
| 556 |
+
return env_val
|
| 557 |
+
|
| 558 |
+
# 2. .env file values (with prefix)
|
| 559 |
+
dotenv_val = self._dotenv_data.get(env_key)
|
| 560 |
+
if dotenv_val is not None:
|
| 561 |
+
return dotenv_val
|
| 562 |
+
|
| 563 |
+
# 3. Config file values (exact key first, then nested lookup)
|
| 564 |
+
if self._config_data:
|
| 565 |
+
# Exact key match (e.g. INI-flattened keys)
|
| 566 |
+
if key in self._config_data:
|
| 567 |
+
return self._config_data[key]
|
| 568 |
+
# Nested lookup via separator splitting
|
| 569 |
+
if self._separator in key:
|
| 570 |
+
parts = key.split(self._separator)
|
| 571 |
+
result = _deep_get(self._config_data, parts)
|
| 572 |
+
if result is not MISSING:
|
| 573 |
+
return result
|
| 574 |
+
|
| 575 |
+
return MISSING
|
| 576 |
+
|
| 577 |
+
def get(
|
| 578 |
+
self,
|
| 579 |
+
key: str,
|
| 580 |
+
*,
|
| 581 |
+
default: Any = MISSING,
|
| 582 |
+
cast: type | Callable[..., Any] | None = None,
|
| 583 |
+
) -> Any:
|
| 584 |
+
"""Retrieve a configuration value.
|
| 585 |
+
|
| 586 |
+
Args:
|
| 587 |
+
key: Configuration key to look up.
|
| 588 |
+
default: Default value if key is not found. If not provided and
|
| 589 |
+
the key is missing, ``UndefinedValueError`` is raised.
|
| 590 |
+
cast: Type or callable to apply to the value. Built-in support
|
| 591 |
+
for ``bool``, ``int``, ``float``, ``list``, ``tuple``.
|
| 592 |
+
Also accepts ``Csv(...)`` and ``Choices(...)`` instances.
|
| 593 |
+
|
| 594 |
+
Returns:
|
| 595 |
+
The resolved and optionally cast configuration value.
|
| 596 |
+
|
| 597 |
+
Raises:
|
| 598 |
+
UndefinedValueError: If key is missing and no default is given.
|
| 599 |
+
"""
|
| 600 |
+
value = self._lookup(key)
|
| 601 |
+
|
| 602 |
+
if value is MISSING:
|
| 603 |
+
if default is MISSING:
|
| 604 |
+
raise UndefinedValueError(
|
| 605 |
+
f"{key!r} is not set and has no default value."
|
| 606 |
+
)
|
| 607 |
+
value = default
|
| 608 |
+
|
| 609 |
+
return _apply_cast(value, cast)
|
| 610 |
+
|
| 611 |
+
def __call__(
|
| 612 |
+
self,
|
| 613 |
+
key: str,
|
| 614 |
+
*,
|
| 615 |
+
default: Any = MISSING,
|
| 616 |
+
cast: type | Callable[..., Any] | None = None,
|
| 617 |
+
) -> Any:
|
| 618 |
+
"""Shorthand for ``get()``. See :meth:`get` for details."""
|
| 619 |
+
return self.get(key, default=default, cast=cast)
|
| 620 |
+
|
| 621 |
+
def has(self, key: str) -> bool:
|
| 622 |
+
"""Check whether a key exists in any source."""
|
| 623 |
+
return self._lookup(key) is not MISSING
|
| 624 |
+
|
| 625 |
+
def as_dict(self) -> dict[str, Any]:
|
| 626 |
+
"""Return a merged view of all config sources (flattened).
|
| 627 |
+
|
| 628 |
+
Lower-priority sources are merged first, so higher-priority
|
| 629 |
+
sources overwrite them.
|
| 630 |
+
"""
|
| 631 |
+
merged: dict[str, Any] = {}
|
| 632 |
+
# 3. Config file (lowest priority)
|
| 633 |
+
if self._config_data:
|
| 634 |
+
merged.update(_flatten_dict(self._config_data, self._separator))
|
| 635 |
+
# 2. .env file
|
| 636 |
+
for k, v in self._dotenv_data.items():
|
| 637 |
+
if v is not None:
|
| 638 |
+
merged[k] = v
|
| 639 |
+
# 1. Environment variables (only those matching prefix)
|
| 640 |
+
if self._prefix:
|
| 641 |
+
plen = len(self._prefix)
|
| 642 |
+
for k, v in os.environ.items():
|
| 643 |
+
if k.startswith(self._prefix):
|
| 644 |
+
merged[k[plen:]] = v
|
| 645 |
+
else:
|
| 646 |
+
merged.update(os.environ)
|
| 647 |
+
return merged
|
| 648 |
+
|
| 649 |
+
|
| 650 |
+
# ── Module-level convenience ────────────────────────────────────────────────
|
| 651 |
+
|
| 652 |
+
_default_config: Config | None = None
|
| 653 |
+
|
| 654 |
+
|
| 655 |
+
def setup(
|
| 656 |
+
*,
|
| 657 |
+
dotenv_path: str | os.PathLike[str] | None | _Auto = _AUTO,
|
| 658 |
+
config_path: str | os.PathLike[str] | None = None,
|
| 659 |
+
prefix: str = "",
|
| 660 |
+
separator: str = "__",
|
| 661 |
+
) -> Config:
|
| 662 |
+
"""Initialize the module-level :class:`Config` instance.
|
| 663 |
+
|
| 664 |
+
Call this once at application startup to configure sources and prefix.
|
| 665 |
+
Subsequent calls to :func:`config` will use this instance.
|
| 666 |
+
|
| 667 |
+
Args:
|
| 668 |
+
dotenv_path: Path to ``.env`` file, ``None`` to disable, or
|
| 669 |
+
``_AUTO`` (default) to auto-discover.
|
| 670 |
+
config_path: Path to a config file (JSON/YAML/TOML/INI).
|
| 671 |
+
prefix: Prefix for environment variable lookups.
|
| 672 |
+
separator: Separator for nested key access.
|
| 673 |
+
|
| 674 |
+
Returns:
|
| 675 |
+
The newly created :class:`Config` instance.
|
| 676 |
+
"""
|
| 677 |
+
global _default_config
|
| 678 |
+
_default_config = Config(
|
| 679 |
+
dotenv_path=dotenv_path,
|
| 680 |
+
config_path=config_path,
|
| 681 |
+
prefix=prefix,
|
| 682 |
+
separator=separator,
|
| 683 |
+
)
|
| 684 |
+
return _default_config
|
| 685 |
+
|
| 686 |
+
|
| 687 |
+
def config(
|
| 688 |
+
key: str,
|
| 689 |
+
*,
|
| 690 |
+
default: Any = MISSING,
|
| 691 |
+
cast: type | Callable[..., Any] | None = None,
|
| 692 |
+
) -> Any:
|
| 693 |
+
"""Look up a configuration value using the module-level instance.
|
| 694 |
+
|
| 695 |
+
If :func:`setup` has not been called, a default :class:`Config` is
|
| 696 |
+
created automatically (auto-discovers ``.env``, no config file,
|
| 697 |
+
no prefix).
|
| 698 |
+
|
| 699 |
+
Args:
|
| 700 |
+
key: Configuration key.
|
| 701 |
+
default: Fallback value if missing.
|
| 702 |
+
cast: Type or callable to coerce the value.
|
| 703 |
+
|
| 704 |
+
Returns:
|
| 705 |
+
The resolved configuration value.
|
| 706 |
+
|
| 707 |
+
Raises:
|
| 708 |
+
UndefinedValueError: If key is missing and no default.
|
| 709 |
+
"""
|
| 710 |
+
global _default_config
|
| 711 |
+
if _default_config is None:
|
| 712 |
+
_default_config = Config()
|
| 713 |
+
return _default_config(key, default=default, cast=cast)
|
|
@@ -0,0 +1,514 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.3.1"
|
| 3 |
+
# deps = []
|
| 4 |
+
# tier = "simple"
|
| 5 |
+
# category = "config"
|
| 6 |
+
# note = "Install/update via `zerodep add dotenv`"
|
| 7 |
+
# ///
|
| 8 |
+
|
| 9 |
+
""".env file parser and loader — zero dependencies, stdlib only, Python 3.10+.
|
| 10 |
+
|
| 11 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 12 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 13 |
+
|
| 14 |
+
Drop-in replacement for python-dotenv core functionality.
|
| 15 |
+
|
| 16 |
+
Example::
|
| 17 |
+
|
| 18 |
+
load_dotenv() # load .env into os.environ
|
| 19 |
+
values = dotenv_values(".env") # parse without modifying environ
|
| 20 |
+
path = find_dotenv() # search up for .env file
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import inspect
|
| 26 |
+
import os
|
| 27 |
+
import re
|
| 28 |
+
import sys
|
| 29 |
+
from pathlib import Path
|
| 30 |
+
from typing import IO, Iterator, Mapping
|
| 31 |
+
|
| 32 |
+
__all__ = [
|
| 33 |
+
"find_dotenv",
|
| 34 |
+
"get_key",
|
| 35 |
+
"set_key",
|
| 36 |
+
"unset_key",
|
| 37 |
+
"dotenv_values",
|
| 38 |
+
"load_dotenv",
|
| 39 |
+
]
|
| 40 |
+
|
| 41 |
+
# ── Constants / Regex ──────────────────────────────────────────────────────────
|
| 42 |
+
|
| 43 |
+
_ESCAPE_MAP = {
|
| 44 |
+
"\\": "\\",
|
| 45 |
+
"'": "'",
|
| 46 |
+
'"': '"',
|
| 47 |
+
"n": "\n",
|
| 48 |
+
"r": "\r",
|
| 49 |
+
"t": "\t",
|
| 50 |
+
"$": "$",
|
| 51 |
+
" ": " ",
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
# Matches a single .env binding line. Groups:
|
| 55 |
+
# key – variable name
|
| 56 |
+
# sq – single-quoted value content
|
| 57 |
+
# dq – double-quoted value content (may span multiple lines)
|
| 58 |
+
# uq – unquoted value content
|
| 59 |
+
_BINDING_RE = re.compile(
|
| 60 |
+
r"""
|
| 61 |
+
\A\s* # leading whitespace
|
| 62 |
+
(?:export\s+)? # optional export prefix
|
| 63 |
+
(?P<key>[A-Za-z_]\w*) # key (must start with letter or _)
|
| 64 |
+
(?: # optional =value part
|
| 65 |
+
\s*=\s* # equals sign
|
| 66 |
+
(?:
|
| 67 |
+
'(?P<sq>[^']*)' # single-quoted (no escapes)
|
| 68 |
+
| "(?P<dq>(?:[^"\\]|\\.)*)" # double-quoted (with escapes)
|
| 69 |
+
| (?P<uq>[^\#\r\n]*) # unquoted (up to # or EOL)
|
| 70 |
+
)
|
| 71 |
+
)? # entire =value is optional
|
| 72 |
+
\s* # trailing whitespace
|
| 73 |
+
(?:\#.*)? # optional inline comment
|
| 74 |
+
\s*\Z # end
|
| 75 |
+
""",
|
| 76 |
+
re.VERBOSE,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
_INTERPOLATE_RE = re.compile(
|
| 80 |
+
r"""
|
| 81 |
+
(?<!\\) # not preceded by backslash
|
| 82 |
+
\$ # dollar sign
|
| 83 |
+
(?:
|
| 84 |
+
\{(?P<brace>[^}]*)\} # ${VAR} or ${VAR:-default}
|
| 85 |
+
| (?P<name>[A-Za-z_]\w*) # $VAR
|
| 86 |
+
)
|
| 87 |
+
""",
|
| 88 |
+
re.VERBOSE,
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
# ── Internal helpers ───────────────────────────────────────────────────────────
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _unescape_double_quoted(s: str) -> str:
|
| 96 |
+
"""Process escape sequences in a double-quoted value."""
|
| 97 |
+
|
| 98 |
+
def _replace(m: re.Match[str]) -> str:
|
| 99 |
+
ch = m.group(1)
|
| 100 |
+
return _ESCAPE_MAP.get(ch, "\\" + ch)
|
| 101 |
+
|
| 102 |
+
return re.sub(r"\\(.)", _replace, s)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _interpolate(
|
| 106 |
+
value: str,
|
| 107 |
+
env: dict[str, str | None],
|
| 108 |
+
os_environ: Mapping[str, str],
|
| 109 |
+
) -> str:
|
| 110 |
+
"""Expand $VAR and ${VAR} references in *value*.
|
| 111 |
+
|
| 112 |
+
Resolution order: file-local *env*, then *os_environ*, then empty string.
|
| 113 |
+
Supports ${VAR:-default} syntax.
|
| 114 |
+
"""
|
| 115 |
+
|
| 116 |
+
def _replace(m: re.Match[str]) -> str:
|
| 117 |
+
brace = m.group("brace")
|
| 118 |
+
if brace is not None:
|
| 119 |
+
# Handle ${VAR:-default}
|
| 120 |
+
if ":-" in brace:
|
| 121 |
+
name, default = brace.split(":-", 1)
|
| 122 |
+
name = name.strip()
|
| 123 |
+
val = env.get(name) if name in env else os_environ.get(name)
|
| 124 |
+
return val if val is not None else default
|
| 125 |
+
name = brace.strip()
|
| 126 |
+
else:
|
| 127 |
+
name = m.group("name")
|
| 128 |
+
val = env.get(name) if name in env else os_environ.get(name)
|
| 129 |
+
return val if val is not None else ""
|
| 130 |
+
|
| 131 |
+
return _INTERPOLATE_RE.sub(_replace, value)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def _find_unescaped_quote(text: str) -> int:
|
| 135 |
+
"""Return position of the first unescaped double-quote in *text*, or -1."""
|
| 136 |
+
pos = 0
|
| 137 |
+
while pos < len(text):
|
| 138 |
+
if text[pos] == "\\" and pos + 1 < len(text):
|
| 139 |
+
pos += 2
|
| 140 |
+
continue
|
| 141 |
+
if text[pos] == '"':
|
| 142 |
+
return pos
|
| 143 |
+
pos += 1
|
| 144 |
+
return -1
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _collect_multiline_value(
|
| 148 |
+
lines: list[str], start_idx: int, first_fragment: str
|
| 149 |
+
) -> tuple[str, int]:
|
| 150 |
+
"""Accumulate continuation lines for a multiline double-quoted value.
|
| 151 |
+
|
| 152 |
+
Args:
|
| 153 |
+
lines: All lines of the .env content.
|
| 154 |
+
start_idx: Index of the first continuation line to inspect.
|
| 155 |
+
first_fragment: Text after the opening ``"`` on the first line.
|
| 156 |
+
|
| 157 |
+
Returns:
|
| 158 |
+
Tuple of (raw concatenated value, next line index to process).
|
| 159 |
+
"""
|
| 160 |
+
parts = [first_fragment]
|
| 161 |
+
i = start_idx
|
| 162 |
+
while i < len(lines):
|
| 163 |
+
next_line = lines[i]
|
| 164 |
+
i += 1
|
| 165 |
+
close_pos = _find_unescaped_quote(next_line)
|
| 166 |
+
if close_pos >= 0:
|
| 167 |
+
parts.append(next_line[:close_pos])
|
| 168 |
+
break
|
| 169 |
+
parts.append(next_line)
|
| 170 |
+
return "".join(parts), i
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
_MULTILINE_OPEN_RE = re.compile(
|
| 174 |
+
r'\A\s*(?:export\s+)?([A-Za-z_]\w*)\s*=\s*"',
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _extract_value_from_match(m: re.Match[str]) -> str | None:
|
| 179 |
+
"""Extract the value from a single-line binding regex match.
|
| 180 |
+
|
| 181 |
+
Returns the raw value (unescaped for double-quoted, literal for
|
| 182 |
+
single-quoted, stripped for unquoted, or None when no ``=`` present).
|
| 183 |
+
"""
|
| 184 |
+
if m.group("sq") is not None:
|
| 185 |
+
return m.group("sq")
|
| 186 |
+
if m.group("dq") is not None:
|
| 187 |
+
return _unescape_double_quoted(m.group("dq"))
|
| 188 |
+
if m.group("uq") is not None:
|
| 189 |
+
return m.group("uq").rstrip()
|
| 190 |
+
return None
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _parse_stream(
|
| 194 |
+
stream: IO[str],
|
| 195 |
+
interpolate: bool = True,
|
| 196 |
+
override: bool = False,
|
| 197 |
+
os_environ: Mapping[str, str] | None = None,
|
| 198 |
+
) -> Iterator[tuple[str, str | None]]:
|
| 199 |
+
"""Parse an .env stream, yielding (key, value) tuples."""
|
| 200 |
+
if os_environ is None:
|
| 201 |
+
os_environ = os.environ
|
| 202 |
+
|
| 203 |
+
env: dict[str, str | None] = {}
|
| 204 |
+
content = stream.read()
|
| 205 |
+
|
| 206 |
+
# Strip UTF-8 BOM
|
| 207 |
+
if content.startswith("\ufeff"):
|
| 208 |
+
content = content[1:]
|
| 209 |
+
|
| 210 |
+
lines = content.splitlines(keepends=True)
|
| 211 |
+
i = 0
|
| 212 |
+
|
| 213 |
+
while i < len(lines):
|
| 214 |
+
line = lines[i]
|
| 215 |
+
stripped = line.strip()
|
| 216 |
+
i += 1
|
| 217 |
+
|
| 218 |
+
# Skip blank lines and comment-only lines
|
| 219 |
+
if not stripped or stripped.startswith("#"):
|
| 220 |
+
continue
|
| 221 |
+
|
| 222 |
+
# Check for multiline double-quoted values
|
| 223 |
+
ml_match = _MULTILINE_OPEN_RE.match(line)
|
| 224 |
+
if ml_match:
|
| 225 |
+
after_eq_quote = line[ml_match.end() :]
|
| 226 |
+
if _find_unescaped_quote(after_eq_quote) < 0:
|
| 227 |
+
# Multiline: accumulate lines until closing "
|
| 228 |
+
key = ml_match.group(1)
|
| 229 |
+
raw_value, i = _collect_multiline_value(lines, i, after_eq_quote)
|
| 230 |
+
value = _unescape_double_quoted(raw_value)
|
| 231 |
+
if interpolate:
|
| 232 |
+
value = _interpolate(value, env, os_environ)
|
| 233 |
+
env[key] = value
|
| 234 |
+
yield key, value
|
| 235 |
+
continue
|
| 236 |
+
|
| 237 |
+
# Single-line binding
|
| 238 |
+
m = _BINDING_RE.match(stripped)
|
| 239 |
+
if m is None:
|
| 240 |
+
continue
|
| 241 |
+
|
| 242 |
+
key = m.group("key")
|
| 243 |
+
value: str | None = _extract_value_from_match(m)
|
| 244 |
+
|
| 245 |
+
if value is not None and interpolate:
|
| 246 |
+
value = _interpolate(value, env, os_environ)
|
| 247 |
+
|
| 248 |
+
env[key] = value
|
| 249 |
+
yield key, value
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
# ── File search ────────────────────────────────────────────────────────────────
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def find_dotenv(
|
| 256 |
+
filename: str = ".env",
|
| 257 |
+
raise_error_if_not_found: bool = False,
|
| 258 |
+
usecwd: bool = False,
|
| 259 |
+
) -> str:
|
| 260 |
+
"""Walk up from the calling file's directory (or cwd) to find *filename*.
|
| 261 |
+
|
| 262 |
+
Args:
|
| 263 |
+
filename: Name of the file to search for.
|
| 264 |
+
raise_error_if_not_found: Raise IOError if the file is not found.
|
| 265 |
+
usecwd: Start from the current working directory instead of the
|
| 266 |
+
calling file's directory.
|
| 267 |
+
|
| 268 |
+
Returns:
|
| 269 |
+
Absolute path to the found file, or empty string if not found.
|
| 270 |
+
"""
|
| 271 |
+
if usecwd:
|
| 272 |
+
start = Path.cwd()
|
| 273 |
+
else:
|
| 274 |
+
frame = inspect.currentframe()
|
| 275 |
+
caller = frame.f_back if frame is not None else None
|
| 276 |
+
if caller is not None and caller.f_globals.get("__file__"):
|
| 277 |
+
start = Path(caller.f_globals["__file__"]).resolve().parent
|
| 278 |
+
else:
|
| 279 |
+
start = Path.cwd()
|
| 280 |
+
|
| 281 |
+
current = start
|
| 282 |
+
while True:
|
| 283 |
+
candidate = current / filename
|
| 284 |
+
if candidate.is_file():
|
| 285 |
+
return str(candidate)
|
| 286 |
+
parent = current.parent
|
| 287 |
+
if parent == current:
|
| 288 |
+
break
|
| 289 |
+
current = parent
|
| 290 |
+
|
| 291 |
+
if raise_error_if_not_found:
|
| 292 |
+
raise IOError(f"File {filename!r} not found starting from {start}")
|
| 293 |
+
return ""
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
# ── File I/O (set / unset) ────────────────────────────────────────────────────
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def get_key(
|
| 300 |
+
dotenv_path: str | os.PathLike[str],
|
| 301 |
+
key_to_get: str,
|
| 302 |
+
encoding: str = "utf-8",
|
| 303 |
+
) -> str | None:
|
| 304 |
+
"""Get a single value from a .env file.
|
| 305 |
+
|
| 306 |
+
Args:
|
| 307 |
+
dotenv_path: Path to the .env file.
|
| 308 |
+
key_to_get: The key to retrieve.
|
| 309 |
+
encoding: File encoding.
|
| 310 |
+
|
| 311 |
+
Returns:
|
| 312 |
+
The value for the key, or None if not found.
|
| 313 |
+
"""
|
| 314 |
+
values = dotenv_values(dotenv_path, encoding=encoding)
|
| 315 |
+
return values.get(key_to_get)
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def set_key(
|
| 319 |
+
dotenv_path: str | os.PathLike[str],
|
| 320 |
+
key_to_set: str,
|
| 321 |
+
value_to_set: str,
|
| 322 |
+
quote_mode: str = "always",
|
| 323 |
+
export: bool = False,
|
| 324 |
+
encoding: str = "utf-8",
|
| 325 |
+
) -> tuple[bool, str, str]:
|
| 326 |
+
"""Set a key=value pair in a .env file, creating it if needed.
|
| 327 |
+
|
| 328 |
+
Args:
|
| 329 |
+
dotenv_path: Path to the .env file.
|
| 330 |
+
key_to_set: The key to set.
|
| 331 |
+
value_to_set: The value to set.
|
| 332 |
+
quote_mode: Quoting strategy: "always", "auto", or "never".
|
| 333 |
+
export: Whether to prefix with ``export``.
|
| 334 |
+
encoding: File encoding.
|
| 335 |
+
|
| 336 |
+
Returns:
|
| 337 |
+
Tuple of (success, key, value).
|
| 338 |
+
"""
|
| 339 |
+
path = Path(dotenv_path)
|
| 340 |
+
|
| 341 |
+
if quote_mode == "always":
|
| 342 |
+
value_out = f'"{value_to_set}"'
|
| 343 |
+
elif quote_mode == "never":
|
| 344 |
+
value_out = value_to_set
|
| 345 |
+
else:
|
| 346 |
+
# auto: quote if value contains spaces or special chars
|
| 347 |
+
if re.search(r"[\s#\"']", value_to_set):
|
| 348 |
+
value_out = f'"{value_to_set}"'
|
| 349 |
+
else:
|
| 350 |
+
value_out = value_to_set
|
| 351 |
+
|
| 352 |
+
export_prefix = "export " if export else ""
|
| 353 |
+
new_line = f"{export_prefix}{key_to_set}={value_out}\n"
|
| 354 |
+
|
| 355 |
+
if path.is_file():
|
| 356 |
+
lines = path.read_text(encoding=encoding).splitlines(keepends=True)
|
| 357 |
+
replaced = False
|
| 358 |
+
for idx, line in enumerate(lines):
|
| 359 |
+
m = re.match(
|
| 360 |
+
r"\A\s*(?:export\s+)?(" + re.escape(key_to_set) + r")\s*=",
|
| 361 |
+
line,
|
| 362 |
+
)
|
| 363 |
+
if m:
|
| 364 |
+
lines[idx] = new_line
|
| 365 |
+
replaced = True
|
| 366 |
+
break
|
| 367 |
+
if not replaced:
|
| 368 |
+
if lines and not lines[-1].endswith("\n"):
|
| 369 |
+
lines[-1] += "\n"
|
| 370 |
+
lines.append(new_line)
|
| 371 |
+
path.write_text("".join(lines), encoding=encoding)
|
| 372 |
+
else:
|
| 373 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 374 |
+
path.write_text(new_line, encoding=encoding)
|
| 375 |
+
|
| 376 |
+
return True, key_to_set, value_to_set
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def unset_key(
|
| 380 |
+
dotenv_path: str | os.PathLike[str],
|
| 381 |
+
key_to_unset: str,
|
| 382 |
+
quote_mode: str = "always",
|
| 383 |
+
encoding: str = "utf-8",
|
| 384 |
+
) -> tuple[bool, str]:
|
| 385 |
+
"""Remove a key from a .env file.
|
| 386 |
+
|
| 387 |
+
Args:
|
| 388 |
+
dotenv_path: Path to the .env file.
|
| 389 |
+
key_to_unset: The key to remove.
|
| 390 |
+
quote_mode: Unused, kept for API compatibility.
|
| 391 |
+
encoding: File encoding.
|
| 392 |
+
|
| 393 |
+
Returns:
|
| 394 |
+
Tuple of (success, key).
|
| 395 |
+
"""
|
| 396 |
+
path = Path(dotenv_path)
|
| 397 |
+
if not path.is_file():
|
| 398 |
+
return True, key_to_unset
|
| 399 |
+
|
| 400 |
+
lines = path.read_text(encoding=encoding).splitlines(keepends=True)
|
| 401 |
+
new_lines = []
|
| 402 |
+
for line in lines:
|
| 403 |
+
m = re.match(
|
| 404 |
+
r"\A\s*(?:export\s+)?(" + re.escape(key_to_unset) + r")\s*=",
|
| 405 |
+
line,
|
| 406 |
+
)
|
| 407 |
+
if m:
|
| 408 |
+
continue
|
| 409 |
+
new_lines.append(line)
|
| 410 |
+
|
| 411 |
+
path.write_text("".join(new_lines), encoding=encoding)
|
| 412 |
+
return True, key_to_unset
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
# ── Public API ─────────────────────────────────────────────────────────────────
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
def _resolve_dotenv_path(
|
| 419 |
+
dotenv_path: str | os.PathLike[str] | None,
|
| 420 |
+
verbose: bool,
|
| 421 |
+
) -> Path | None:
|
| 422 |
+
"""Resolve *dotenv_path* to an existing file, or return None.
|
| 423 |
+
|
| 424 |
+
When *verbose* is True and the file does not exist, a warning is printed.
|
| 425 |
+
"""
|
| 426 |
+
if dotenv_path is None:
|
| 427 |
+
dotenv_path = find_dotenv(usecwd=True)
|
| 428 |
+
|
| 429 |
+
path = Path(dotenv_path)
|
| 430 |
+
if path.is_file():
|
| 431 |
+
return path
|
| 432 |
+
|
| 433 |
+
if verbose:
|
| 434 |
+
print( # noqa: T201
|
| 435 |
+
f"Python-dotenv could not find configuration file {path}.",
|
| 436 |
+
file=sys.stderr,
|
| 437 |
+
)
|
| 438 |
+
return None
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
def _apply_to_environ(
|
| 442 |
+
pairs: Iterator[tuple[str, str | None]],
|
| 443 |
+
override: bool,
|
| 444 |
+
) -> None:
|
| 445 |
+
"""Write parsed (key, value) pairs into ``os.environ``."""
|
| 446 |
+
for key, value in pairs:
|
| 447 |
+
if value is not None and (override or key not in os.environ):
|
| 448 |
+
os.environ[key] = value
|
| 449 |
+
|
| 450 |
+
|
| 451 |
+
def dotenv_values(
|
| 452 |
+
dotenv_path: str | os.PathLike[str] | None = None,
|
| 453 |
+
stream: IO[str] | None = None,
|
| 454 |
+
verbose: bool = False,
|
| 455 |
+
interpolate: bool = True,
|
| 456 |
+
override: bool = False,
|
| 457 |
+
encoding: str = "utf-8",
|
| 458 |
+
) -> dict[str, str | None]:
|
| 459 |
+
"""Parse a .env file and return a dict without modifying ``os.environ``.
|
| 460 |
+
|
| 461 |
+
Args:
|
| 462 |
+
dotenv_path: Path to the .env file. If None, uses ``find_dotenv()``.
|
| 463 |
+
stream: A text stream to read from (overrides *dotenv_path*).
|
| 464 |
+
verbose: Print a warning when the file is missing.
|
| 465 |
+
interpolate: Expand ``$VAR`` and ``${VAR}`` references.
|
| 466 |
+
override: Unused for dotenv_values (kept for API compatibility).
|
| 467 |
+
encoding: File encoding.
|
| 468 |
+
|
| 469 |
+
Returns:
|
| 470 |
+
Dictionary mapping variable names to their values.
|
| 471 |
+
"""
|
| 472 |
+
if stream is not None:
|
| 473 |
+
return dict(_parse_stream(stream, interpolate=interpolate))
|
| 474 |
+
|
| 475 |
+
path = _resolve_dotenv_path(dotenv_path, verbose)
|
| 476 |
+
if path is None:
|
| 477 |
+
return {}
|
| 478 |
+
|
| 479 |
+
with open(path, encoding=encoding) as f:
|
| 480 |
+
return dict(_parse_stream(f, interpolate=interpolate))
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
def load_dotenv(
|
| 484 |
+
dotenv_path: str | os.PathLike[str] | None = None,
|
| 485 |
+
stream: IO[str] | None = None,
|
| 486 |
+
verbose: bool = False,
|
| 487 |
+
interpolate: bool = True,
|
| 488 |
+
override: bool = False,
|
| 489 |
+
encoding: str = "utf-8",
|
| 490 |
+
) -> bool:
|
| 491 |
+
"""Read a .env file and set ``os.environ``.
|
| 492 |
+
|
| 493 |
+
Args:
|
| 494 |
+
dotenv_path: Path to the .env file. If None, uses ``find_dotenv()``.
|
| 495 |
+
stream: A text stream to read from (overrides *dotenv_path*).
|
| 496 |
+
verbose: Print a warning when the file is missing.
|
| 497 |
+
interpolate: Expand ``$VAR`` and ``${VAR}`` references.
|
| 498 |
+
override: If True, overwrite existing environment variables.
|
| 499 |
+
encoding: File encoding.
|
| 500 |
+
|
| 501 |
+
Returns:
|
| 502 |
+
True if a file was found and loaded.
|
| 503 |
+
"""
|
| 504 |
+
if stream is not None:
|
| 505 |
+
_apply_to_environ(_parse_stream(stream, interpolate=interpolate), override)
|
| 506 |
+
return True
|
| 507 |
+
|
| 508 |
+
path = _resolve_dotenv_path(dotenv_path, verbose)
|
| 509 |
+
if path is None:
|
| 510 |
+
return False
|
| 511 |
+
|
| 512 |
+
with open(path, encoding=encoding) as f:
|
| 513 |
+
_apply_to_environ(_parse_stream(f, interpolate=interpolate), override)
|
| 514 |
+
return True
|
|
@@ -0,0 +1,1007 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.1.0"
|
| 3 |
+
# deps = []
|
| 4 |
+
# tier = "subsystem"
|
| 5 |
+
# category = "network"
|
| 6 |
+
# note = "Install/update via `zerodep add httpserver`"
|
| 7 |
+
# ///
|
| 8 |
+
|
| 9 |
+
"""Zero-dependency async HTTP server with decorator-based routing.
|
| 10 |
+
|
| 11 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 12 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 13 |
+
|
| 14 |
+
Async HTTP/1.1 server built on ``asyncio.start_server()``. Supports
|
| 15 |
+
decorator-based routing, JSON request/response, static file serving,
|
| 16 |
+
streaming responses (SSE), and graceful shutdown.
|
| 17 |
+
|
| 18 |
+
Usage::
|
| 19 |
+
|
| 20 |
+
from httpserver import App, JSONResponse
|
| 21 |
+
|
| 22 |
+
app = App()
|
| 23 |
+
|
| 24 |
+
@app.route("/status")
|
| 25 |
+
async def status(request):
|
| 26 |
+
return JSONResponse({"state": "idle"})
|
| 27 |
+
|
| 28 |
+
@app.route("/echo", methods=["POST"])
|
| 29 |
+
def echo(request):
|
| 30 |
+
return JSONResponse(request.json())
|
| 31 |
+
|
| 32 |
+
app.run(host="127.0.0.1", port=8000)
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
# ── Imports ──────────────────────────────────────────────────────────────────
|
| 36 |
+
|
| 37 |
+
from __future__ import annotations
|
| 38 |
+
|
| 39 |
+
import asyncio
|
| 40 |
+
import dataclasses
|
| 41 |
+
import inspect
|
| 42 |
+
import json as _json
|
| 43 |
+
import logging
|
| 44 |
+
import mimetypes
|
| 45 |
+
import os
|
| 46 |
+
import re
|
| 47 |
+
import signal
|
| 48 |
+
import sys
|
| 49 |
+
from collections.abc import AsyncIterator, Callable
|
| 50 |
+
from email.utils import formatdate
|
| 51 |
+
from pathlib import Path
|
| 52 |
+
from typing import Any
|
| 53 |
+
from urllib.parse import parse_qs, unquote, urlparse
|
| 54 |
+
|
| 55 |
+
__all__ = [
|
| 56 |
+
# Application
|
| 57 |
+
"App",
|
| 58 |
+
# Request / Response
|
| 59 |
+
"Request",
|
| 60 |
+
"Response",
|
| 61 |
+
"JSONResponse",
|
| 62 |
+
"StreamingResponse",
|
| 63 |
+
"FileResponse",
|
| 64 |
+
# Exceptions
|
| 65 |
+
"HTTPException",
|
| 66 |
+
# Utilities
|
| 67 |
+
"abort",
|
| 68 |
+
]
|
| 69 |
+
|
| 70 |
+
logger = logging.getLogger(__name__)
|
| 71 |
+
|
| 72 |
+
# ── Constants ────────────────────────────────────────────────────────────────
|
| 73 |
+
|
| 74 |
+
DEFAULT_HOST = "127.0.0.1"
|
| 75 |
+
DEFAULT_PORT = 8000
|
| 76 |
+
DEFAULT_MAX_BODY_SIZE = 1_048_576 # 1 MB
|
| 77 |
+
DEFAULT_READ_TIMEOUT = 30.0 # seconds
|
| 78 |
+
|
| 79 |
+
_STATUS_REASONS: dict[int, str] = {
|
| 80 |
+
100: "Continue",
|
| 81 |
+
101: "Switching Protocols",
|
| 82 |
+
200: "OK",
|
| 83 |
+
201: "Created",
|
| 84 |
+
202: "Accepted",
|
| 85 |
+
204: "No Content",
|
| 86 |
+
301: "Moved Permanently",
|
| 87 |
+
302: "Found",
|
| 88 |
+
303: "See Other",
|
| 89 |
+
304: "Not Modified",
|
| 90 |
+
307: "Temporary Redirect",
|
| 91 |
+
308: "Permanent Redirect",
|
| 92 |
+
400: "Bad Request",
|
| 93 |
+
401: "Unauthorized",
|
| 94 |
+
403: "Forbidden",
|
| 95 |
+
404: "Not Found",
|
| 96 |
+
405: "Method Not Allowed",
|
| 97 |
+
406: "Not Acceptable",
|
| 98 |
+
408: "Request Timeout",
|
| 99 |
+
409: "Conflict",
|
| 100 |
+
413: "Content Too Large",
|
| 101 |
+
415: "Unsupported Media Type",
|
| 102 |
+
422: "Unprocessable Entity",
|
| 103 |
+
429: "Too Many Requests",
|
| 104 |
+
500: "Internal Server Error",
|
| 105 |
+
502: "Bad Gateway",
|
| 106 |
+
503: "Service Unavailable",
|
| 107 |
+
504: "Gateway Timeout",
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
# Type converters for path parameters
|
| 111 |
+
_PARAM_CONVERTERS: dict[str, tuple[str, Callable[[str], Any]]] = {
|
| 112 |
+
"str": (r"([^/]+)", str),
|
| 113 |
+
"int": (r"(-?\d+)", int),
|
| 114 |
+
"float": (r"(-?[0-9]+(?:\.[0-9]+)?)", float),
|
| 115 |
+
"path": (r"(.+)", str),
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
# Regex to find <type:name> or <name> segments in route patterns
|
| 119 |
+
_PARAM_RE = re.compile(r"<(?:(\w+):)?(\w+)>")
|
| 120 |
+
|
| 121 |
+
_SENTINEL = object()
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
# ── Exceptions ───────────────────────────────────────────────────────────────
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
class HTTPException(Exception):
|
| 128 |
+
"""HTTP error that maps to a specific status code.
|
| 129 |
+
|
| 130 |
+
Raise directly or via :func:`abort` to short-circuit request handling.
|
| 131 |
+
|
| 132 |
+
Args:
|
| 133 |
+
status_code: HTTP status code (e.g. 404, 500).
|
| 134 |
+
message: Optional human-readable error message.
|
| 135 |
+
"""
|
| 136 |
+
|
| 137 |
+
__slots__ = ("status_code", "message")
|
| 138 |
+
|
| 139 |
+
def __init__(self, status_code: int, message: str | None = None):
|
| 140 |
+
self.status_code = status_code
|
| 141 |
+
self.message = message or _STATUS_REASONS.get(status_code, "Error")
|
| 142 |
+
super().__init__(self.message)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
class _BadRequest(Exception):
|
| 146 |
+
"""Internal: malformed HTTP request from client."""
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def abort(status_code: int, message: str | None = None) -> None:
|
| 150 |
+
"""Raise an :class:`HTTPException` with the given status code.
|
| 151 |
+
|
| 152 |
+
Args:
|
| 153 |
+
status_code: HTTP status code.
|
| 154 |
+
message: Optional error message.
|
| 155 |
+
"""
|
| 156 |
+
raise HTTPException(status_code, message)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
# ── Request ──────────────────────────────────────────────────────────────────
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
class Request:
|
| 163 |
+
"""Parsed HTTP request.
|
| 164 |
+
|
| 165 |
+
Attributes:
|
| 166 |
+
method: Uppercase HTTP method (GET, POST, ...).
|
| 167 |
+
path: URL path, percent-decoded.
|
| 168 |
+
query_string: Raw query string (without leading ``?``).
|
| 169 |
+
query_params: Parsed query string as ``{key: [values]}``.
|
| 170 |
+
headers: Case-insensitive header dict (keys stored lowercase).
|
| 171 |
+
body: Raw request body bytes.
|
| 172 |
+
path_params: Parameters extracted from the route pattern.
|
| 173 |
+
client_addr: Client ``(host, port)`` tuple.
|
| 174 |
+
app: Reference to the :class:`App` instance handling this request.
|
| 175 |
+
"""
|
| 176 |
+
|
| 177 |
+
__slots__ = (
|
| 178 |
+
"method",
|
| 179 |
+
"path",
|
| 180 |
+
"query_string",
|
| 181 |
+
"query_params",
|
| 182 |
+
"headers",
|
| 183 |
+
"body",
|
| 184 |
+
"path_params",
|
| 185 |
+
"client_addr",
|
| 186 |
+
"app",
|
| 187 |
+
"_json",
|
| 188 |
+
)
|
| 189 |
+
|
| 190 |
+
def __init__(
|
| 191 |
+
self,
|
| 192 |
+
method: str,
|
| 193 |
+
path: str,
|
| 194 |
+
query_string: str,
|
| 195 |
+
headers: dict[str, str],
|
| 196 |
+
body: bytes,
|
| 197 |
+
client_addr: tuple[str, int],
|
| 198 |
+
app: App | None = None,
|
| 199 |
+
):
|
| 200 |
+
self.method = method
|
| 201 |
+
self.path = path
|
| 202 |
+
self.query_string = query_string
|
| 203 |
+
self.query_params: dict[str, list[str]] = parse_qs(query_string)
|
| 204 |
+
self.headers = headers
|
| 205 |
+
self.body = body
|
| 206 |
+
self.path_params: dict[str, Any] = {}
|
| 207 |
+
self.client_addr = client_addr
|
| 208 |
+
self.app = app
|
| 209 |
+
self._json: Any = _SENTINEL
|
| 210 |
+
|
| 211 |
+
def json(self) -> Any:
|
| 212 |
+
"""Parse body as JSON (cached)."""
|
| 213 |
+
if self._json is _SENTINEL:
|
| 214 |
+
self._json = _json.loads(self.body)
|
| 215 |
+
return self._json
|
| 216 |
+
|
| 217 |
+
def text(self) -> str:
|
| 218 |
+
"""Decode body as UTF-8."""
|
| 219 |
+
return self.body.decode("utf-8")
|
| 220 |
+
|
| 221 |
+
def form(self) -> dict[str, list[str]]:
|
| 222 |
+
"""Parse URL-encoded form body."""
|
| 223 |
+
return parse_qs(self.body.decode("utf-8"))
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
# ── Response Classes ─────────────────────────────────────────────────────────
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
class Response:
|
| 230 |
+
"""HTTP response with a fixed body.
|
| 231 |
+
|
| 232 |
+
Args:
|
| 233 |
+
body: Response body (bytes or str).
|
| 234 |
+
status_code: HTTP status code.
|
| 235 |
+
headers: Extra response headers.
|
| 236 |
+
content_type: Shorthand for ``Content-Type`` header.
|
| 237 |
+
"""
|
| 238 |
+
|
| 239 |
+
__slots__ = ("status_code", "headers", "body")
|
| 240 |
+
|
| 241 |
+
def __init__(
|
| 242 |
+
self,
|
| 243 |
+
body: bytes | str = b"",
|
| 244 |
+
status_code: int = 200,
|
| 245 |
+
headers: dict[str, str] | None = None,
|
| 246 |
+
content_type: str | None = None,
|
| 247 |
+
):
|
| 248 |
+
self.status_code = status_code
|
| 249 |
+
self.headers: dict[str, str] = headers.copy() if headers else {}
|
| 250 |
+
if isinstance(body, str):
|
| 251 |
+
self.body = body.encode("utf-8")
|
| 252 |
+
else:
|
| 253 |
+
self.body = body
|
| 254 |
+
if content_type is not None:
|
| 255 |
+
self.headers["Content-Type"] = content_type
|
| 256 |
+
|
| 257 |
+
async def _write(self, writer: asyncio.StreamWriter) -> None:
|
| 258 |
+
"""Serialize and write the full HTTP response."""
|
| 259 |
+
reason = _STATUS_REASONS.get(self.status_code, "Unknown")
|
| 260 |
+
self.headers.setdefault("Content-Length", str(len(self.body)))
|
| 261 |
+
self.headers.setdefault("Content-Type", "text/plain; charset=utf-8")
|
| 262 |
+
self.headers.setdefault("Date", _http_date())
|
| 263 |
+
self.headers.setdefault("Connection", "close")
|
| 264 |
+
|
| 265 |
+
buf = bytearray()
|
| 266 |
+
buf.extend(f"HTTP/1.1 {self.status_code} {reason}\r\n".encode("latin-1"))
|
| 267 |
+
for k, v in self.headers.items():
|
| 268 |
+
buf.extend(f"{k}: {v}\r\n".encode("latin-1"))
|
| 269 |
+
buf.extend(b"\r\n")
|
| 270 |
+
buf.extend(self.body)
|
| 271 |
+
writer.write(bytes(buf))
|
| 272 |
+
await writer.drain()
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
class JSONResponse(Response):
|
| 276 |
+
"""Response serialized as JSON.
|
| 277 |
+
|
| 278 |
+
Args:
|
| 279 |
+
data: Python object to serialize.
|
| 280 |
+
status_code: HTTP status code.
|
| 281 |
+
headers: Extra response headers.
|
| 282 |
+
"""
|
| 283 |
+
|
| 284 |
+
def __init__(
|
| 285 |
+
self,
|
| 286 |
+
data: Any,
|
| 287 |
+
status_code: int = 200,
|
| 288 |
+
headers: dict[str, str] | None = None,
|
| 289 |
+
):
|
| 290 |
+
body = _json.dumps(data, ensure_ascii=False).encode("utf-8")
|
| 291 |
+
super().__init__(
|
| 292 |
+
body=body,
|
| 293 |
+
status_code=status_code,
|
| 294 |
+
headers=headers,
|
| 295 |
+
content_type="application/json; charset=utf-8",
|
| 296 |
+
)
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
class StreamingResponse:
|
| 300 |
+
"""HTTP response streamed from an async generator.
|
| 301 |
+
|
| 302 |
+
Writes chunks using ``Transfer-Encoding: chunked`` unless
|
| 303 |
+
``content_type`` is ``text/event-stream`` (SSE), in which case raw
|
| 304 |
+
bytes are flushed directly for maximum compatibility with SSE clients.
|
| 305 |
+
|
| 306 |
+
Args:
|
| 307 |
+
generator: Async iterator yielding ``bytes`` or ``str`` chunks.
|
| 308 |
+
status_code: HTTP status code.
|
| 309 |
+
headers: Extra response headers.
|
| 310 |
+
content_type: MIME type (default ``application/octet-stream``).
|
| 311 |
+
"""
|
| 312 |
+
|
| 313 |
+
__slots__ = ("_generator", "status_code", "headers", "content_type")
|
| 314 |
+
|
| 315 |
+
def __init__(
|
| 316 |
+
self,
|
| 317 |
+
generator: AsyncIterator[bytes | str],
|
| 318 |
+
status_code: int = 200,
|
| 319 |
+
headers: dict[str, str] | None = None,
|
| 320 |
+
content_type: str = "application/octet-stream",
|
| 321 |
+
):
|
| 322 |
+
self._generator = generator
|
| 323 |
+
self.status_code = status_code
|
| 324 |
+
self.headers: dict[str, str] = headers.copy() if headers else {}
|
| 325 |
+
self.content_type = content_type
|
| 326 |
+
|
| 327 |
+
async def _write(self, writer: asyncio.StreamWriter) -> None:
|
| 328 |
+
"""Write status line, headers, then stream the body."""
|
| 329 |
+
reason = _STATUS_REASONS.get(self.status_code, "Unknown")
|
| 330 |
+
is_sse = self.content_type.startswith("text/event-stream")
|
| 331 |
+
|
| 332 |
+
self.headers["Content-Type"] = self.content_type
|
| 333 |
+
if not is_sse:
|
| 334 |
+
self.headers.setdefault("Transfer-Encoding", "chunked")
|
| 335 |
+
else:
|
| 336 |
+
self.headers.setdefault("Cache-Control", "no-cache")
|
| 337 |
+
self.headers.setdefault("Date", _http_date())
|
| 338 |
+
self.headers.setdefault("Connection", "close")
|
| 339 |
+
|
| 340 |
+
buf = bytearray()
|
| 341 |
+
buf.extend(f"HTTP/1.1 {self.status_code} {reason}\r\n".encode("latin-1"))
|
| 342 |
+
for k, v in self.headers.items():
|
| 343 |
+
buf.extend(f"{k}: {v}\r\n".encode("latin-1"))
|
| 344 |
+
buf.extend(b"\r\n")
|
| 345 |
+
writer.write(bytes(buf))
|
| 346 |
+
await writer.drain()
|
| 347 |
+
|
| 348 |
+
try:
|
| 349 |
+
async for chunk in self._generator:
|
| 350 |
+
if isinstance(chunk, str):
|
| 351 |
+
chunk = chunk.encode("utf-8")
|
| 352 |
+
if is_sse:
|
| 353 |
+
writer.write(chunk)
|
| 354 |
+
else:
|
| 355 |
+
writer.write(f"{len(chunk):x}\r\n".encode("latin-1"))
|
| 356 |
+
writer.write(chunk)
|
| 357 |
+
writer.write(b"\r\n")
|
| 358 |
+
await writer.drain()
|
| 359 |
+
if not is_sse:
|
| 360 |
+
writer.write(b"0\r\n\r\n")
|
| 361 |
+
await writer.drain()
|
| 362 |
+
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
|
| 363 |
+
logger.debug("Client disconnected during streaming")
|
| 364 |
+
finally:
|
| 365 |
+
aclose = getattr(self._generator, "aclose", None)
|
| 366 |
+
if aclose is not None:
|
| 367 |
+
await aclose()
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
class FileResponse(Response):
|
| 371 |
+
"""Response serving a file from disk.
|
| 372 |
+
|
| 373 |
+
Args:
|
| 374 |
+
path: Path to the file.
|
| 375 |
+
content_type: MIME type (auto-detected from extension if ``None``).
|
| 376 |
+
status_code: HTTP status code.
|
| 377 |
+
"""
|
| 378 |
+
|
| 379 |
+
def __init__(
|
| 380 |
+
self,
|
| 381 |
+
path: str | Path,
|
| 382 |
+
content_type: str | None = None,
|
| 383 |
+
status_code: int = 200,
|
| 384 |
+
):
|
| 385 |
+
file_path = Path(path)
|
| 386 |
+
body = file_path.read_bytes()
|
| 387 |
+
if content_type is None:
|
| 388 |
+
guessed, _ = mimetypes.guess_type(str(file_path))
|
| 389 |
+
content_type = guessed or "application/octet-stream"
|
| 390 |
+
headers = {
|
| 391 |
+
"Last-Modified": _http_date(os.path.getmtime(file_path)),
|
| 392 |
+
}
|
| 393 |
+
super().__init__(
|
| 394 |
+
body=body,
|
| 395 |
+
status_code=status_code,
|
| 396 |
+
headers=headers,
|
| 397 |
+
content_type=content_type,
|
| 398 |
+
)
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
# ── Routing ──────────────────────────────────────────────────────────────────
|
| 402 |
+
|
| 403 |
+
|
| 404 |
+
@dataclasses.dataclass(frozen=True, slots=True)
|
| 405 |
+
class _Route:
|
| 406 |
+
"""Internal route entry."""
|
| 407 |
+
|
| 408 |
+
methods: list[str] # uppercase, e.g. ["GET", "POST"], or ["*"]
|
| 409 |
+
pattern: re.Pattern[str]
|
| 410 |
+
handler: Callable[..., Any]
|
| 411 |
+
is_async: bool
|
| 412 |
+
param_names: list[str]
|
| 413 |
+
param_converters: list[Callable[[str], Any]]
|
| 414 |
+
|
| 415 |
+
|
| 416 |
+
def _compile_route(
|
| 417 |
+
path: str,
|
| 418 |
+
) -> tuple[re.Pattern[str], list[str], list[Callable[[str], Any]]]:
|
| 419 |
+
"""Compile a path pattern into a regex and parameter metadata.
|
| 420 |
+
|
| 421 |
+
Supports ``<name>``, ``<int:name>``, ``<float:name>``, ``<path:name>``.
|
| 422 |
+
|
| 423 |
+
Args:
|
| 424 |
+
path: URL path pattern (e.g. ``/users/<int:id>``).
|
| 425 |
+
|
| 426 |
+
Returns:
|
| 427 |
+
Tuple of (compiled regex, param names, param converters).
|
| 428 |
+
"""
|
| 429 |
+
param_names: list[str] = []
|
| 430 |
+
param_converters: list[Callable[[str], Any]] = []
|
| 431 |
+
regex_parts: list[str] = []
|
| 432 |
+
last_end = 0
|
| 433 |
+
|
| 434 |
+
for m in _PARAM_RE.finditer(path):
|
| 435 |
+
regex_parts.append(re.escape(path[last_end : m.start()]))
|
| 436 |
+
type_name = m.group(1) or "str"
|
| 437 |
+
name = m.group(2)
|
| 438 |
+
if type_name not in _PARAM_CONVERTERS:
|
| 439 |
+
raise ValueError(f"Unknown path parameter type: {type_name!r}")
|
| 440 |
+
regex_fragment, converter = _PARAM_CONVERTERS[type_name]
|
| 441 |
+
regex_parts.append(regex_fragment)
|
| 442 |
+
param_names.append(name)
|
| 443 |
+
param_converters.append(converter)
|
| 444 |
+
last_end = m.end()
|
| 445 |
+
|
| 446 |
+
regex_parts.append(re.escape(path[last_end:]))
|
| 447 |
+
full_regex = "^" + "".join(regex_parts) + "$"
|
| 448 |
+
return re.compile(full_regex), param_names, param_converters
|
| 449 |
+
|
| 450 |
+
|
| 451 |
+
# ── HTTP Parsing ─────────────────────────────────────────────────────────────
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
async def _read_request(
|
| 455 |
+
reader: asyncio.StreamReader,
|
| 456 |
+
timeout: float,
|
| 457 |
+
max_body_size: int,
|
| 458 |
+
) -> tuple[str, str, str, dict[str, str], bytes]:
|
| 459 |
+
"""Read and parse an HTTP request from a stream.
|
| 460 |
+
|
| 461 |
+
Returns:
|
| 462 |
+
Tuple of (method, path, query_string, headers, body).
|
| 463 |
+
|
| 464 |
+
Raises:
|
| 465 |
+
_BadRequest: If the request is malformed.
|
| 466 |
+
asyncio.TimeoutError: If reading times out.
|
| 467 |
+
"""
|
| 468 |
+
# -- Request line --
|
| 469 |
+
raw_line = await asyncio.wait_for(reader.readline(), timeout=timeout)
|
| 470 |
+
if not raw_line:
|
| 471 |
+
raise _BadRequest("Empty request")
|
| 472 |
+
request_line = raw_line.decode("latin-1").rstrip("\r\n")
|
| 473 |
+
parts = request_line.split(" ", 2)
|
| 474 |
+
if len(parts) != 3:
|
| 475 |
+
raise _BadRequest(f"Malformed request line: {request_line!r}")
|
| 476 |
+
method, raw_url, _version = parts
|
| 477 |
+
|
| 478 |
+
# -- URL --
|
| 479 |
+
parsed = urlparse(raw_url)
|
| 480 |
+
path = unquote(parsed.path)
|
| 481 |
+
query_string = parsed.query
|
| 482 |
+
|
| 483 |
+
# -- Headers --
|
| 484 |
+
headers: dict[str, str] = {}
|
| 485 |
+
while True:
|
| 486 |
+
line = await asyncio.wait_for(reader.readline(), timeout=timeout)
|
| 487 |
+
decoded = line.decode("latin-1").rstrip("\r\n")
|
| 488 |
+
if not decoded:
|
| 489 |
+
break
|
| 490 |
+
if ":" in decoded:
|
| 491 |
+
k, v = decoded.split(":", 1)
|
| 492 |
+
headers[k.strip().lower()] = v.strip()
|
| 493 |
+
|
| 494 |
+
# -- Body --
|
| 495 |
+
body = b""
|
| 496 |
+
cl = headers.get("content-length")
|
| 497 |
+
if cl is not None:
|
| 498 |
+
length = int(cl)
|
| 499 |
+
if length > max_body_size:
|
| 500 |
+
raise HTTPException(413, f"Request body too large ({length} bytes)")
|
| 501 |
+
if length > 0:
|
| 502 |
+
body = await asyncio.wait_for(reader.readexactly(length), timeout=timeout)
|
| 503 |
+
elif headers.get("transfer-encoding", "").lower() == "chunked":
|
| 504 |
+
body = await _read_chunked_body(reader, timeout, max_body_size)
|
| 505 |
+
|
| 506 |
+
return method.upper(), path, query_string, headers, body
|
| 507 |
+
|
| 508 |
+
|
| 509 |
+
async def _read_chunked_body(
|
| 510 |
+
reader: asyncio.StreamReader,
|
| 511 |
+
timeout: float,
|
| 512 |
+
max_body_size: int,
|
| 513 |
+
) -> bytes:
|
| 514 |
+
"""Read a chunked transfer-encoded body."""
|
| 515 |
+
parts: list[bytes] = []
|
| 516 |
+
total = 0
|
| 517 |
+
while True:
|
| 518 |
+
size_line = await asyncio.wait_for(reader.readline(), timeout=timeout)
|
| 519 |
+
size_str = size_line.decode("latin-1").split(";")[0].strip()
|
| 520 |
+
if not size_str:
|
| 521 |
+
break
|
| 522 |
+
chunk_size = int(size_str, 16)
|
| 523 |
+
if chunk_size == 0:
|
| 524 |
+
await asyncio.wait_for(reader.readline(), timeout=timeout)
|
| 525 |
+
break
|
| 526 |
+
total += chunk_size
|
| 527 |
+
if total > max_body_size:
|
| 528 |
+
raise HTTPException(413, "Request body too large")
|
| 529 |
+
data = await asyncio.wait_for(reader.readexactly(chunk_size), timeout=timeout)
|
| 530 |
+
await asyncio.wait_for(reader.readline(), timeout=timeout)
|
| 531 |
+
parts.append(data)
|
| 532 |
+
return b"".join(parts)
|
| 533 |
+
|
| 534 |
+
|
| 535 |
+
# ── Utilities ────────────────────────────────────────────────────────────────
|
| 536 |
+
|
| 537 |
+
|
| 538 |
+
def _http_date(timestamp: float | None = None) -> str:
|
| 539 |
+
"""Format a timestamp as an HTTP-date (RFC 7231)."""
|
| 540 |
+
return formatdate(timeval=timestamp, localtime=False, usegmt=True)
|
| 541 |
+
|
| 542 |
+
|
| 543 |
+
def _coerce_response(result: Any) -> Response | StreamingResponse:
|
| 544 |
+
"""Convert a handler return value into a Response object."""
|
| 545 |
+
if isinstance(result, (Response, StreamingResponse)):
|
| 546 |
+
return result
|
| 547 |
+
|
| 548 |
+
if result is None:
|
| 549 |
+
return Response(status_code=204)
|
| 550 |
+
|
| 551 |
+
if isinstance(result, dict):
|
| 552 |
+
return JSONResponse(result)
|
| 553 |
+
|
| 554 |
+
if isinstance(result, tuple):
|
| 555 |
+
if len(result) == 2:
|
| 556 |
+
body, status = result
|
| 557 |
+
resp = _coerce_response(body)
|
| 558 |
+
resp.status_code = status
|
| 559 |
+
return resp
|
| 560 |
+
if len(result) == 3:
|
| 561 |
+
body, status, extra_headers = result
|
| 562 |
+
resp = _coerce_response(body)
|
| 563 |
+
resp.status_code = status
|
| 564 |
+
resp.headers.update(extra_headers)
|
| 565 |
+
return resp
|
| 566 |
+
raise ValueError(f"Unsupported tuple length: {len(result)}")
|
| 567 |
+
|
| 568 |
+
if isinstance(result, bytes):
|
| 569 |
+
return Response(body=result, content_type="application/octet-stream")
|
| 570 |
+
|
| 571 |
+
if isinstance(result, str):
|
| 572 |
+
return Response(body=result, content_type="text/plain; charset=utf-8")
|
| 573 |
+
|
| 574 |
+
raise TypeError(
|
| 575 |
+
f"Cannot coerce handler return type {type(result).__name__} to Response"
|
| 576 |
+
)
|
| 577 |
+
|
| 578 |
+
|
| 579 |
+
# ── Static File Resolution ───────────────────────────────────────────────────
|
| 580 |
+
|
| 581 |
+
|
| 582 |
+
def _resolve_static_file(
|
| 583 |
+
request_path: str,
|
| 584 |
+
url_prefix: str,
|
| 585 |
+
directory: str,
|
| 586 |
+
) -> FileResponse | None:
|
| 587 |
+
"""Try to resolve a static file request.
|
| 588 |
+
|
| 589 |
+
Returns a FileResponse if the file exists and the path is safe,
|
| 590 |
+
otherwise None.
|
| 591 |
+
"""
|
| 592 |
+
if not request_path.startswith(url_prefix):
|
| 593 |
+
return None
|
| 594 |
+
|
| 595 |
+
relative = request_path[len(url_prefix) :].lstrip("/")
|
| 596 |
+
if not relative:
|
| 597 |
+
return None
|
| 598 |
+
|
| 599 |
+
base = Path(directory).resolve()
|
| 600 |
+
target = (base / relative).resolve()
|
| 601 |
+
|
| 602 |
+
# Prevent directory traversal
|
| 603 |
+
if not str(target).startswith(str(base)):
|
| 604 |
+
return None
|
| 605 |
+
|
| 606 |
+
if not target.is_file():
|
| 607 |
+
return None
|
| 608 |
+
|
| 609 |
+
return FileResponse(target)
|
| 610 |
+
|
| 611 |
+
|
| 612 |
+
# ── App ──────────────────────────────────────────────────────────────────────
|
| 613 |
+
|
| 614 |
+
|
| 615 |
+
class App:
|
| 616 |
+
"""Async HTTP server application.
|
| 617 |
+
|
| 618 |
+
Args:
|
| 619 |
+
max_body_size: Maximum request body size in bytes.
|
| 620 |
+
read_timeout: Timeout for reading a single request (seconds).
|
| 621 |
+
|
| 622 |
+
Example::
|
| 623 |
+
|
| 624 |
+
app = App()
|
| 625 |
+
|
| 626 |
+
@app.get("/hello")
|
| 627 |
+
async def hello(request):
|
| 628 |
+
return {"message": "Hello, world!"}
|
| 629 |
+
|
| 630 |
+
app.run()
|
| 631 |
+
"""
|
| 632 |
+
|
| 633 |
+
def __init__(
|
| 634 |
+
self,
|
| 635 |
+
*,
|
| 636 |
+
max_body_size: int = DEFAULT_MAX_BODY_SIZE,
|
| 637 |
+
read_timeout: float = DEFAULT_READ_TIMEOUT,
|
| 638 |
+
):
|
| 639 |
+
self._routes: list[_Route] = []
|
| 640 |
+
self._static_routes: list[tuple[str, str]] = []
|
| 641 |
+
self._before_request_handlers: list[Callable[..., Any]] = []
|
| 642 |
+
self._after_request_handlers: list[Callable[..., Any]] = []
|
| 643 |
+
self._error_handlers: dict[int | type, Callable[..., Any]] = {}
|
| 644 |
+
self._server: asyncio.Server | None = None
|
| 645 |
+
self._shutdown_event: asyncio.Event | None = None
|
| 646 |
+
self.max_body_size = max_body_size
|
| 647 |
+
self.read_timeout = read_timeout
|
| 648 |
+
self.port: int | None = None
|
| 649 |
+
self.host: str | None = None
|
| 650 |
+
|
| 651 |
+
# ── Route Registration ───────────────────────────────────────────────
|
| 652 |
+
|
| 653 |
+
def route(
|
| 654 |
+
self, url_pattern: str, methods: list[str] | None = None
|
| 655 |
+
) -> Callable[..., Any]:
|
| 656 |
+
"""Register a route handler.
|
| 657 |
+
|
| 658 |
+
Args:
|
| 659 |
+
url_pattern: URL pattern with optional parameters.
|
| 660 |
+
methods: HTTP methods to handle (e.g. ``["GET", "POST"]``).
|
| 661 |
+
Defaults to ``["GET"]``.
|
| 662 |
+
|
| 663 |
+
Example::
|
| 664 |
+
|
| 665 |
+
@app.route("/users/<int:id>", methods=["GET", "POST"])
|
| 666 |
+
async def user(request, id):
|
| 667 |
+
return {"id": id}
|
| 668 |
+
"""
|
| 669 |
+
if methods is None:
|
| 670 |
+
methods = ["GET"]
|
| 671 |
+
|
| 672 |
+
def decorator(handler: Callable[..., Any]) -> Callable[..., Any]:
|
| 673 |
+
pattern, param_names, converters = _compile_route(url_pattern)
|
| 674 |
+
is_async = inspect.iscoroutinefunction(handler)
|
| 675 |
+
self._routes.append(
|
| 676 |
+
_Route(
|
| 677 |
+
methods=[m.upper() for m in methods],
|
| 678 |
+
pattern=pattern,
|
| 679 |
+
handler=handler,
|
| 680 |
+
is_async=is_async,
|
| 681 |
+
param_names=param_names,
|
| 682 |
+
param_converters=converters,
|
| 683 |
+
)
|
| 684 |
+
)
|
| 685 |
+
return handler
|
| 686 |
+
|
| 687 |
+
return decorator
|
| 688 |
+
|
| 689 |
+
def get(self, path: str) -> Callable[..., Any]:
|
| 690 |
+
"""Shorthand for ``@app.route(path, methods=["GET"])``."""
|
| 691 |
+
return self.route(path, methods=["GET"])
|
| 692 |
+
|
| 693 |
+
def post(self, path: str) -> Callable[..., Any]:
|
| 694 |
+
"""Shorthand for ``@app.route(path, methods=["POST"])``."""
|
| 695 |
+
return self.route(path, methods=["POST"])
|
| 696 |
+
|
| 697 |
+
def put(self, path: str) -> Callable[..., Any]:
|
| 698 |
+
"""Shorthand for ``@app.route(path, methods=["PUT"])``."""
|
| 699 |
+
return self.route(path, methods=["PUT"])
|
| 700 |
+
|
| 701 |
+
def delete(self, path: str) -> Callable[..., Any]:
|
| 702 |
+
"""Shorthand for ``@app.route(path, methods=["DELETE"])``."""
|
| 703 |
+
return self.route(path, methods=["DELETE"])
|
| 704 |
+
|
| 705 |
+
def patch(self, path: str) -> Callable[..., Any]:
|
| 706 |
+
"""Shorthand for ``@app.route(path, methods=["PATCH"])``."""
|
| 707 |
+
return self.route(path, methods=["PATCH"])
|
| 708 |
+
|
| 709 |
+
def static(self, url_prefix: str, directory: str) -> None:
|
| 710 |
+
"""Register a directory for static file serving.
|
| 711 |
+
|
| 712 |
+
Args:
|
| 713 |
+
url_prefix: URL prefix (e.g. ``"/static"``).
|
| 714 |
+
directory: Filesystem path to the directory.
|
| 715 |
+
|
| 716 |
+
Example::
|
| 717 |
+
|
| 718 |
+
app.static("/assets", "./public")
|
| 719 |
+
"""
|
| 720 |
+
url_prefix = url_prefix.rstrip("/")
|
| 721 |
+
abs_dir = os.path.abspath(directory)
|
| 722 |
+
self._static_routes.append((url_prefix, abs_dir))
|
| 723 |
+
|
| 724 |
+
# ── Middleware ────────────────────────────────────────────────────────
|
| 725 |
+
|
| 726 |
+
def before_request(self, handler: Callable[..., Any]) -> Callable[..., Any]:
|
| 727 |
+
"""Register a before-request hook.
|
| 728 |
+
|
| 729 |
+
The hook receives ``(request)`` and may return a ``Response`` to
|
| 730 |
+
short-circuit the route handler.
|
| 731 |
+
"""
|
| 732 |
+
self._before_request_handlers.append(handler)
|
| 733 |
+
return handler
|
| 734 |
+
|
| 735 |
+
def after_request(self, handler: Callable[..., Any]) -> Callable[..., Any]:
|
| 736 |
+
"""Register an after-request hook.
|
| 737 |
+
|
| 738 |
+
The hook receives ``(request, response)`` and may return a new
|
| 739 |
+
or modified ``Response``.
|
| 740 |
+
"""
|
| 741 |
+
self._after_request_handlers.append(handler)
|
| 742 |
+
return handler
|
| 743 |
+
|
| 744 |
+
def errorhandler(self, code_or_exc: int | type) -> Callable[..., Any]:
|
| 745 |
+
"""Register an error handler.
|
| 746 |
+
|
| 747 |
+
Args:
|
| 748 |
+
code_or_exc: HTTP status code (int) or exception class.
|
| 749 |
+
|
| 750 |
+
The handler receives ``(request, exception)`` and must return a
|
| 751 |
+
``Response``.
|
| 752 |
+
"""
|
| 753 |
+
|
| 754 |
+
def decorator(handler: Callable[..., Any]) -> Callable[..., Any]:
|
| 755 |
+
self._error_handlers[code_or_exc] = handler
|
| 756 |
+
return handler
|
| 757 |
+
|
| 758 |
+
return decorator
|
| 759 |
+
|
| 760 |
+
# ── Request Dispatch ─────────────────────────────────────────────────
|
| 761 |
+
|
| 762 |
+
def _match_route(
|
| 763 |
+
self, request: Request
|
| 764 |
+
) -> tuple[_Route | None, re.Match[str] | None, bool]:
|
| 765 |
+
"""Find the first route matching the request path and method.
|
| 766 |
+
|
| 767 |
+
Returns:
|
| 768 |
+
(matched_route, regex_match, path_existed). *path_existed* is
|
| 769 |
+
True when a route matched the path but not the HTTP method.
|
| 770 |
+
"""
|
| 771 |
+
path_existed = False
|
| 772 |
+
for route in self._routes:
|
| 773 |
+
m = route.pattern.match(request.path)
|
| 774 |
+
if m is None:
|
| 775 |
+
continue
|
| 776 |
+
path_existed = True
|
| 777 |
+
if "*" not in route.methods and request.method not in route.methods:
|
| 778 |
+
continue
|
| 779 |
+
return route, m, True
|
| 780 |
+
return None, None, path_existed
|
| 781 |
+
|
| 782 |
+
async def _invoke_route(
|
| 783 |
+
self, route: _Route, match: re.Match[str], request: Request
|
| 784 |
+
) -> Response | StreamingResponse:
|
| 785 |
+
"""Extract path params, call the handler, return a response."""
|
| 786 |
+
for name, converter, value in zip(
|
| 787 |
+
route.param_names, route.param_converters, match.groups()
|
| 788 |
+
):
|
| 789 |
+
request.path_params[name] = converter(value)
|
| 790 |
+
|
| 791 |
+
try:
|
| 792 |
+
result = await _invoke(route.handler, request, **request.path_params)
|
| 793 |
+
return _coerce_response(result)
|
| 794 |
+
except HTTPException as exc:
|
| 795 |
+
return await self._handle_error(request, exc)
|
| 796 |
+
except Exception as exc:
|
| 797 |
+
logger.exception(
|
| 798 |
+
"Unhandled exception in handler %s",
|
| 799 |
+
getattr(route.handler, "__name__", repr(route.handler)),
|
| 800 |
+
)
|
| 801 |
+
return await self._handle_error(request, exc)
|
| 802 |
+
|
| 803 |
+
async def _run_after_hooks(
|
| 804 |
+
self, request: Request, response: Response | StreamingResponse
|
| 805 |
+
) -> Response | StreamingResponse:
|
| 806 |
+
"""Run after-request hooks, allowing them to replace the response."""
|
| 807 |
+
for hook in self._after_request_handlers:
|
| 808 |
+
hook_result = await _invoke(hook, request, response)
|
| 809 |
+
if hook_result is not None:
|
| 810 |
+
response = _coerce_response(hook_result)
|
| 811 |
+
return response
|
| 812 |
+
|
| 813 |
+
async def _dispatch(self, request: Request) -> Response | StreamingResponse:
|
| 814 |
+
"""Match a request to a route and invoke the handler."""
|
| 815 |
+
# -- before_request hooks --
|
| 816 |
+
for hook in self._before_request_handlers:
|
| 817 |
+
result = await _invoke(hook, request)
|
| 818 |
+
if result is not None:
|
| 819 |
+
return _coerce_response(result)
|
| 820 |
+
|
| 821 |
+
# -- Static file check --
|
| 822 |
+
for prefix, directory in self._static_routes:
|
| 823 |
+
file_resp = _resolve_static_file(request.path, prefix, directory)
|
| 824 |
+
if file_resp is not None:
|
| 825 |
+
return file_resp
|
| 826 |
+
|
| 827 |
+
# -- Route matching --
|
| 828 |
+
route, match, path_existed = self._match_route(request)
|
| 829 |
+
|
| 830 |
+
if route is not None and match is not None:
|
| 831 |
+
response = await self._invoke_route(route, match, request)
|
| 832 |
+
return await self._run_after_hooks(request, response)
|
| 833 |
+
|
| 834 |
+
# -- No route matched --
|
| 835 |
+
if path_existed:
|
| 836 |
+
allowed = sorted(
|
| 837 |
+
{
|
| 838 |
+
m
|
| 839 |
+
for r in self._routes
|
| 840 |
+
if r.pattern.match(request.path)
|
| 841 |
+
for m in r.methods
|
| 842 |
+
if m != "*"
|
| 843 |
+
}
|
| 844 |
+
)
|
| 845 |
+
exc = HTTPException(405, "Method Not Allowed")
|
| 846 |
+
response = await self._handle_error(request, exc)
|
| 847 |
+
if isinstance(response, Response):
|
| 848 |
+
response.headers["Allow"] = ", ".join(allowed)
|
| 849 |
+
return response
|
| 850 |
+
|
| 851 |
+
exc = HTTPException(404, "Not Found")
|
| 852 |
+
return await self._handle_error(request, exc)
|
| 853 |
+
|
| 854 |
+
async def _handle_error(
|
| 855 |
+
self, request: Request, exc: Exception
|
| 856 |
+
) -> Response | StreamingResponse:
|
| 857 |
+
"""Resolve an error into a response, consulting registered handlers."""
|
| 858 |
+
if isinstance(exc, HTTPException):
|
| 859 |
+
handler = self._error_handlers.get(exc.status_code)
|
| 860 |
+
if handler is not None:
|
| 861 |
+
result = await _invoke(handler, request, exc)
|
| 862 |
+
return _coerce_response(result)
|
| 863 |
+
|
| 864 |
+
for exc_cls, handler in self._error_handlers.items():
|
| 865 |
+
if isinstance(exc_cls, type) and isinstance(exc, exc_cls):
|
| 866 |
+
result = await _invoke(handler, request, exc)
|
| 867 |
+
return _coerce_response(result)
|
| 868 |
+
|
| 869 |
+
if isinstance(exc, HTTPException):
|
| 870 |
+
return JSONResponse(
|
| 871 |
+
{"error": exc.message},
|
| 872 |
+
status_code=exc.status_code,
|
| 873 |
+
)
|
| 874 |
+
return JSONResponse(
|
| 875 |
+
{"error": "Internal Server Error"},
|
| 876 |
+
status_code=500,
|
| 877 |
+
)
|
| 878 |
+
|
| 879 |
+
# ── Connection Handling ─────────��─────────────────────────────────────
|
| 880 |
+
|
| 881 |
+
async def _handle_connection(
|
| 882 |
+
self,
|
| 883 |
+
reader: asyncio.StreamReader,
|
| 884 |
+
writer: asyncio.StreamWriter,
|
| 885 |
+
) -> None:
|
| 886 |
+
"""Handle a single client connection."""
|
| 887 |
+
peer = writer.get_extra_info("peername")
|
| 888 |
+
client_addr = (peer[0], peer[1]) if peer else ("unknown", 0)
|
| 889 |
+
|
| 890 |
+
try:
|
| 891 |
+
method, path, qs, headers, body = await _read_request(
|
| 892 |
+
reader, self.read_timeout, self.max_body_size
|
| 893 |
+
)
|
| 894 |
+
except _BadRequest as exc:
|
| 895 |
+
logger.debug("Bad request from %s: %s", client_addr, exc)
|
| 896 |
+
response = Response(
|
| 897 |
+
body=str(exc),
|
| 898 |
+
status_code=400,
|
| 899 |
+
content_type="text/plain; charset=utf-8",
|
| 900 |
+
)
|
| 901 |
+
try:
|
| 902 |
+
await response._write(writer)
|
| 903 |
+
except (BrokenPipeError, ConnectionResetError):
|
| 904 |
+
pass
|
| 905 |
+
writer.close()
|
| 906 |
+
return
|
| 907 |
+
except HTTPException as exc:
|
| 908 |
+
response = JSONResponse({"error": exc.message}, status_code=exc.status_code)
|
| 909 |
+
try:
|
| 910 |
+
await response._write(writer)
|
| 911 |
+
except (BrokenPipeError, ConnectionResetError):
|
| 912 |
+
pass
|
| 913 |
+
writer.close()
|
| 914 |
+
return
|
| 915 |
+
except (asyncio.TimeoutError, asyncio.IncompleteReadError):
|
| 916 |
+
writer.close()
|
| 917 |
+
return
|
| 918 |
+
except Exception:
|
| 919 |
+
writer.close()
|
| 920 |
+
return
|
| 921 |
+
|
| 922 |
+
request = Request(
|
| 923 |
+
method=method,
|
| 924 |
+
path=path,
|
| 925 |
+
query_string=qs,
|
| 926 |
+
headers=headers,
|
| 927 |
+
body=body,
|
| 928 |
+
client_addr=client_addr,
|
| 929 |
+
app=self,
|
| 930 |
+
)
|
| 931 |
+
|
| 932 |
+
logger.debug("%s %s from %s", method, path, client_addr)
|
| 933 |
+
|
| 934 |
+
try:
|
| 935 |
+
response = await self._dispatch(request)
|
| 936 |
+
await response._write(writer)
|
| 937 |
+
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
|
| 938 |
+
logger.debug("Connection reset by %s during response", client_addr)
|
| 939 |
+
except Exception:
|
| 940 |
+
logger.exception("Error writing response to %s", client_addr)
|
| 941 |
+
finally:
|
| 942 |
+
try:
|
| 943 |
+
writer.close()
|
| 944 |
+
await writer.wait_closed()
|
| 945 |
+
except Exception:
|
| 946 |
+
pass
|
| 947 |
+
|
| 948 |
+
# ── Server Lifecycle ─────────────────────────────────────────────────
|
| 949 |
+
|
| 950 |
+
def run(
|
| 951 |
+
self,
|
| 952 |
+
host: str = DEFAULT_HOST,
|
| 953 |
+
port: int = DEFAULT_PORT,
|
| 954 |
+
) -> None:
|
| 955 |
+
"""Start the server (blocking).
|
| 956 |
+
|
| 957 |
+
Args:
|
| 958 |
+
host: Bind address.
|
| 959 |
+
port: Bind port. Use ``0`` for OS-assigned port.
|
| 960 |
+
"""
|
| 961 |
+
try:
|
| 962 |
+
asyncio.run(self._serve(host, port))
|
| 963 |
+
except KeyboardInterrupt:
|
| 964 |
+
pass
|
| 965 |
+
|
| 966 |
+
async def _serve(self, host: str, port: int) -> None:
|
| 967 |
+
"""Internal async server loop."""
|
| 968 |
+
self._shutdown_event = asyncio.Event()
|
| 969 |
+
|
| 970 |
+
server = await asyncio.start_server(
|
| 971 |
+
self._handle_connection,
|
| 972 |
+
host,
|
| 973 |
+
port,
|
| 974 |
+
)
|
| 975 |
+
|
| 976 |
+
self._server = server
|
| 977 |
+
addrs = server.sockets[0].getsockname() if server.sockets else (host, port)
|
| 978 |
+
self.host = addrs[0]
|
| 979 |
+
self.port = addrs[1]
|
| 980 |
+
logger.info("Serving on %s:%d", self.host, self.port)
|
| 981 |
+
|
| 982 |
+
loop = asyncio.get_running_loop()
|
| 983 |
+
if sys.platform != "win32":
|
| 984 |
+
for sig in (signal.SIGINT, signal.SIGTERM):
|
| 985 |
+
loop.add_signal_handler(sig, self._shutdown_event.set)
|
| 986 |
+
|
| 987 |
+
async with server:
|
| 988 |
+
await self._shutdown_event.wait()
|
| 989 |
+
logger.info("Shutting down server")
|
| 990 |
+
|
| 991 |
+
def shutdown(self) -> None:
|
| 992 |
+
"""Request a graceful server shutdown.
|
| 993 |
+
|
| 994 |
+
Safe to call from a request handler.
|
| 995 |
+
"""
|
| 996 |
+
if self._shutdown_event is not None:
|
| 997 |
+
self._shutdown_event.set()
|
| 998 |
+
|
| 999 |
+
|
| 1000 |
+
# ── Handler Invocation ───────────────────────────────────────────────────────
|
| 1001 |
+
|
| 1002 |
+
|
| 1003 |
+
async def _invoke(handler: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
|
| 1004 |
+
"""Invoke a handler, wrapping sync functions with ``asyncio.to_thread``."""
|
| 1005 |
+
if inspect.iscoroutinefunction(handler):
|
| 1006 |
+
return await handler(*args, **kwargs)
|
| 1007 |
+
return await asyncio.to_thread(handler, *args, **kwargs)
|
|
@@ -0,0 +1,352 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.3.0"
|
| 3 |
+
# deps = []
|
| 4 |
+
# tier = "simple"
|
| 5 |
+
# category = "serialization"
|
| 6 |
+
# note = "Install/update via `zerodep add jsonc`"
|
| 7 |
+
# ///
|
| 8 |
+
|
| 9 |
+
"""JSONC (JSON with Comments) parser — zero dependencies, stdlib only, Python 3.10+.
|
| 10 |
+
|
| 11 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 12 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 13 |
+
|
| 14 |
+
Drop-in replacement for commentjson / stdlib json with JSONC support.
|
| 15 |
+
|
| 16 |
+
Supports:
|
| 17 |
+
- Single-line comments: ``//`` and ``#``
|
| 18 |
+
- Block comments: ``/* ... */``
|
| 19 |
+
- Trailing commas in objects and arrays
|
| 20 |
+
- All standard JSON types
|
| 21 |
+
|
| 22 |
+
Example::
|
| 23 |
+
|
| 24 |
+
loads('{"a": 1, // comment\n"b": 2}')
|
| 25 |
+
# {'a': 1, 'b': 2}
|
| 26 |
+
load(open("config.jsonc"))
|
| 27 |
+
# {...}
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
from __future__ import annotations
|
| 31 |
+
|
| 32 |
+
import json
|
| 33 |
+
import re
|
| 34 |
+
from typing import IO, Any
|
| 35 |
+
|
| 36 |
+
__all__ = [
|
| 37 |
+
"JSONCDecodeError",
|
| 38 |
+
"loads",
|
| 39 |
+
"load",
|
| 40 |
+
"dumps",
|
| 41 |
+
"dump",
|
| 42 |
+
]
|
| 43 |
+
|
| 44 |
+
# ── Comment stripping ─────────────────────────────────────────────────────────
|
| 45 |
+
|
| 46 |
+
# Matches (in priority order):
|
| 47 |
+
# 1. Double-quoted strings (group 1) — preserved as-is
|
| 48 |
+
# 2. Single-line // comments — removed
|
| 49 |
+
# 3. Single-line # comments — removed
|
| 50 |
+
# 4. Block /* ... */ comments — removed
|
| 51 |
+
_COMMENT_RE = re.compile(
|
| 52 |
+
r"""
|
| 53 |
+
( "(?:[^"\\]|\\.)*" ) # group 1: double-quoted string (keep)
|
| 54 |
+
| //[^\n]* # single-line // comment
|
| 55 |
+
| \#[^\n]* # single-line # comment
|
| 56 |
+
| /\*[\s\S]*?\*/ # block /* */ comment
|
| 57 |
+
""",
|
| 58 |
+
re.VERBOSE | re.MULTILINE,
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
# Trailing comma before closing ] or }
|
| 62 |
+
_TRAILING_COMMA_RE = re.compile(
|
| 63 |
+
r"""
|
| 64 |
+
( "(?:[^"\\]|\\.)*" ) # group 1: double-quoted string (keep)
|
| 65 |
+
| ,\s*(?=[}\]]) # comma followed by optional whitespace then } or ]
|
| 66 |
+
""",
|
| 67 |
+
re.VERBOSE,
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _strip_comments(text: str) -> str:
|
| 72 |
+
"""Remove ``//``, ``#``, and ``/* */`` comments, preserving strings."""
|
| 73 |
+
|
| 74 |
+
def _replace(m: re.Match[str]) -> str:
|
| 75 |
+
if m.group(1) is not None:
|
| 76 |
+
return m.group(1)
|
| 77 |
+
return ""
|
| 78 |
+
|
| 79 |
+
return _COMMENT_RE.sub(_replace, text)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _strip_trailing_commas(text: str) -> str:
|
| 83 |
+
"""Remove trailing commas before ``}`` and ``]``, preserving strings."""
|
| 84 |
+
|
| 85 |
+
def _replace(m: re.Match[str]) -> str:
|
| 86 |
+
if m.group(1) is not None:
|
| 87 |
+
return m.group(1)
|
| 88 |
+
return ""
|
| 89 |
+
|
| 90 |
+
return _TRAILING_COMMA_RE.sub(_replace, text)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _preprocess(text: str) -> str:
|
| 94 |
+
"""Strip comments and trailing commas from JSONC text."""
|
| 95 |
+
text = _strip_comments(text)
|
| 96 |
+
text = _strip_trailing_commas(text)
|
| 97 |
+
return text
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
# ── Line-number error mapping ────────────────────────────────────────────────
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _remap_error_position(
|
| 104 |
+
original: str, cleaned: str, clean_pos: int
|
| 105 |
+
) -> tuple[int, int]:
|
| 106 |
+
"""Map a character offset in *cleaned* text back to *original* line/col.
|
| 107 |
+
|
| 108 |
+
Returns (line, column), both 1-based.
|
| 109 |
+
"""
|
| 110 |
+
# Build a mapping from cleaned-offset → original-offset by replaying the
|
| 111 |
+
# comment-stripping regex. Each matched region in the original either maps
|
| 112 |
+
# 1:1 (strings) or collapses to zero length (comments).
|
| 113 |
+
orig_idx = 0
|
| 114 |
+
clean_idx = 0
|
| 115 |
+
mapping: list[tuple[int, int, int]] = [] # (clean_start, clean_end, orig_start)
|
| 116 |
+
|
| 117 |
+
for m in _COMMENT_RE.finditer(original):
|
| 118 |
+
# Characters before this match map 1:1
|
| 119 |
+
pre_len = m.start() - orig_idx
|
| 120 |
+
if pre_len > 0:
|
| 121 |
+
mapping.append((clean_idx, clean_idx + pre_len, orig_idx))
|
| 122 |
+
clean_idx += pre_len
|
| 123 |
+
orig_idx = m.start()
|
| 124 |
+
|
| 125 |
+
if m.group(1) is not None:
|
| 126 |
+
# Preserved string — maps 1:1
|
| 127 |
+
span_len = m.end() - m.start()
|
| 128 |
+
mapping.append((clean_idx, clean_idx + span_len, orig_idx))
|
| 129 |
+
clean_idx += span_len
|
| 130 |
+
# else: comment — removed, clean_idx doesn't advance
|
| 131 |
+
|
| 132 |
+
orig_idx = m.end()
|
| 133 |
+
|
| 134 |
+
# Tail after last match
|
| 135 |
+
tail_len = len(original) - orig_idx
|
| 136 |
+
if tail_len > 0:
|
| 137 |
+
mapping.append((clean_idx, clean_idx + tail_len, orig_idx))
|
| 138 |
+
|
| 139 |
+
# Look up clean_pos in the mapping
|
| 140 |
+
orig_offset = clean_pos # fallback
|
| 141 |
+
for cs, ce, os_ in mapping:
|
| 142 |
+
if cs <= clean_pos < ce:
|
| 143 |
+
orig_offset = os_ + (clean_pos - cs)
|
| 144 |
+
break
|
| 145 |
+
if clean_pos < cs:
|
| 146 |
+
orig_offset = os_
|
| 147 |
+
break
|
| 148 |
+
|
| 149 |
+
# Convert original offset to line/col
|
| 150 |
+
line = original[:orig_offset].count("\n") + 1
|
| 151 |
+
last_nl = original.rfind("\n", 0, orig_offset)
|
| 152 |
+
col = orig_offset - last_nl # 1-based
|
| 153 |
+
return line, col
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
class JSONCDecodeError(json.JSONDecodeError):
|
| 157 |
+
"""Error raised when JSONC parsing fails.
|
| 158 |
+
|
| 159 |
+
Provides line and column numbers relative to the original JSONC source
|
| 160 |
+
(before comment/trailing-comma stripping).
|
| 161 |
+
"""
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
# ── Public API ────────────────────────────────────────────────────────────────
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def loads(
|
| 168 |
+
text: str,
|
| 169 |
+
*,
|
| 170 |
+
cls: type[json.JSONDecoder] | None = None,
|
| 171 |
+
object_hook: Any = None,
|
| 172 |
+
parse_float: Any = None,
|
| 173 |
+
parse_int: Any = None,
|
| 174 |
+
parse_constant: Any = None,
|
| 175 |
+
object_pairs_hook: Any = None,
|
| 176 |
+
**kwargs: Any,
|
| 177 |
+
) -> Any:
|
| 178 |
+
"""Deserialize a JSONC string to a Python object.
|
| 179 |
+
|
| 180 |
+
Strips ``//``, ``#``, and ``/* */`` comments and trailing commas before
|
| 181 |
+
delegating to :func:`json.loads`.
|
| 182 |
+
|
| 183 |
+
Args:
|
| 184 |
+
text: JSONC source string.
|
| 185 |
+
cls: Custom JSON decoder class.
|
| 186 |
+
object_hook: Called with the result of any object literal decoded.
|
| 187 |
+
parse_float: Called with every JSON float string decoded.
|
| 188 |
+
parse_int: Called with every JSON int string decoded.
|
| 189 |
+
parse_constant: Called with ``-Infinity``, ``Infinity``, ``NaN``.
|
| 190 |
+
object_pairs_hook: Called with an ordered list of pairs.
|
| 191 |
+
**kwargs: Additional keyword arguments passed to :func:`json.loads`.
|
| 192 |
+
|
| 193 |
+
Returns:
|
| 194 |
+
Deserialized Python object.
|
| 195 |
+
|
| 196 |
+
Raises:
|
| 197 |
+
JSONCDecodeError: If the text is not valid JSONC.
|
| 198 |
+
"""
|
| 199 |
+
cleaned = _preprocess(text)
|
| 200 |
+
try:
|
| 201 |
+
return json.loads(
|
| 202 |
+
cleaned,
|
| 203 |
+
cls=cls,
|
| 204 |
+
object_hook=object_hook,
|
| 205 |
+
parse_float=parse_float,
|
| 206 |
+
parse_int=parse_int,
|
| 207 |
+
parse_constant=parse_constant,
|
| 208 |
+
object_pairs_hook=object_pairs_hook,
|
| 209 |
+
**kwargs,
|
| 210 |
+
)
|
| 211 |
+
except json.JSONDecodeError as exc:
|
| 212 |
+
line, col = _remap_error_position(text, cleaned, exc.pos)
|
| 213 |
+
raise JSONCDecodeError(exc.msg, text, exc.pos) from None
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def load(
|
| 217 |
+
fp: IO[str],
|
| 218 |
+
*,
|
| 219 |
+
cls: type[json.JSONDecoder] | None = None,
|
| 220 |
+
object_hook: Any = None,
|
| 221 |
+
parse_float: Any = None,
|
| 222 |
+
parse_int: Any = None,
|
| 223 |
+
parse_constant: Any = None,
|
| 224 |
+
object_pairs_hook: Any = None,
|
| 225 |
+
**kwargs: Any,
|
| 226 |
+
) -> Any:
|
| 227 |
+
"""Deserialize a JSONC file to a Python object.
|
| 228 |
+
|
| 229 |
+
Args:
|
| 230 |
+
fp: A text file-like object containing JSONC.
|
| 231 |
+
cls: Custom JSON decoder class.
|
| 232 |
+
object_hook: Called with the result of any object literal decoded.
|
| 233 |
+
parse_float: Called with every JSON float string decoded.
|
| 234 |
+
parse_int: Called with every JSON int string decoded.
|
| 235 |
+
parse_constant: Called with ``-Infinity``, ``Infinity``, ``NaN``.
|
| 236 |
+
object_pairs_hook: Called with an ordered list of pairs.
|
| 237 |
+
**kwargs: Additional keyword arguments passed to :func:`json.loads`.
|
| 238 |
+
|
| 239 |
+
Returns:
|
| 240 |
+
Deserialized Python object.
|
| 241 |
+
|
| 242 |
+
Raises:
|
| 243 |
+
JSONCDecodeError: If the content is not valid JSONC.
|
| 244 |
+
"""
|
| 245 |
+
return loads(
|
| 246 |
+
fp.read(),
|
| 247 |
+
cls=cls,
|
| 248 |
+
object_hook=object_hook,
|
| 249 |
+
parse_float=parse_float,
|
| 250 |
+
parse_int=parse_int,
|
| 251 |
+
parse_constant=parse_constant,
|
| 252 |
+
object_pairs_hook=object_pairs_hook,
|
| 253 |
+
**kwargs,
|
| 254 |
+
)
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def dumps(
|
| 258 |
+
obj: Any,
|
| 259 |
+
*,
|
| 260 |
+
skipkeys: bool = False,
|
| 261 |
+
ensure_ascii: bool = True,
|
| 262 |
+
check_circular: bool = True,
|
| 263 |
+
allow_nan: bool = True,
|
| 264 |
+
cls: type[json.JSONEncoder] | None = None,
|
| 265 |
+
indent: int | str | None = None,
|
| 266 |
+
separators: tuple[str, str] | None = None,
|
| 267 |
+
default: Any = None,
|
| 268 |
+
sort_keys: bool = False,
|
| 269 |
+
**kwargs: Any,
|
| 270 |
+
) -> str:
|
| 271 |
+
"""Serialize a Python object to a JSON string.
|
| 272 |
+
|
| 273 |
+
This is a pass-through to :func:`json.dumps` for API compatibility.
|
| 274 |
+
|
| 275 |
+
Args:
|
| 276 |
+
obj: Python object to serialize.
|
| 277 |
+
skipkeys: Skip keys that are not basic types.
|
| 278 |
+
ensure_ascii: Escape non-ASCII characters.
|
| 279 |
+
check_circular: Check for circular references.
|
| 280 |
+
allow_nan: Allow ``NaN``, ``Infinity``, ``-Infinity``.
|
| 281 |
+
cls: Custom JSON encoder class.
|
| 282 |
+
indent: Indentation level for pretty-printing.
|
| 283 |
+
separators: Item and key separators.
|
| 284 |
+
default: Called for objects that are not serializable.
|
| 285 |
+
sort_keys: Sort dictionary keys.
|
| 286 |
+
**kwargs: Additional keyword arguments passed to :func:`json.dumps`.
|
| 287 |
+
|
| 288 |
+
Returns:
|
| 289 |
+
JSON string.
|
| 290 |
+
"""
|
| 291 |
+
return json.dumps(
|
| 292 |
+
obj,
|
| 293 |
+
skipkeys=skipkeys,
|
| 294 |
+
ensure_ascii=ensure_ascii,
|
| 295 |
+
check_circular=check_circular,
|
| 296 |
+
allow_nan=allow_nan,
|
| 297 |
+
cls=cls,
|
| 298 |
+
indent=indent,
|
| 299 |
+
separators=separators,
|
| 300 |
+
default=default,
|
| 301 |
+
sort_keys=sort_keys,
|
| 302 |
+
**kwargs,
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def dump(
|
| 307 |
+
obj: Any,
|
| 308 |
+
fp: IO[str],
|
| 309 |
+
*,
|
| 310 |
+
skipkeys: bool = False,
|
| 311 |
+
ensure_ascii: bool = True,
|
| 312 |
+
check_circular: bool = True,
|
| 313 |
+
allow_nan: bool = True,
|
| 314 |
+
cls: type[json.JSONEncoder] | None = None,
|
| 315 |
+
indent: int | str | None = None,
|
| 316 |
+
separators: tuple[str, str] | None = None,
|
| 317 |
+
default: Any = None,
|
| 318 |
+
sort_keys: bool = False,
|
| 319 |
+
**kwargs: Any,
|
| 320 |
+
) -> None:
|
| 321 |
+
"""Serialize a Python object to a JSON file.
|
| 322 |
+
|
| 323 |
+
This is a pass-through to :func:`json.dump` for API compatibility.
|
| 324 |
+
|
| 325 |
+
Args:
|
| 326 |
+
obj: Python object to serialize.
|
| 327 |
+
fp: A text file-like object to write to.
|
| 328 |
+
skipkeys: Skip keys that are not basic types.
|
| 329 |
+
ensure_ascii: Escape non-ASCII characters.
|
| 330 |
+
check_circular: Check for circular references.
|
| 331 |
+
allow_nan: Allow ``NaN``, ``Infinity``, ``-Infinity``.
|
| 332 |
+
cls: Custom JSON encoder class.
|
| 333 |
+
indent: Indentation level for pretty-printing.
|
| 334 |
+
separators: Item and key separators.
|
| 335 |
+
default: Called for objects that are not serializable.
|
| 336 |
+
sort_keys: Sort dictionary keys.
|
| 337 |
+
**kwargs: Additional keyword arguments passed to :func:`json.dump`.
|
| 338 |
+
"""
|
| 339 |
+
json.dump(
|
| 340 |
+
obj,
|
| 341 |
+
fp,
|
| 342 |
+
skipkeys=skipkeys,
|
| 343 |
+
ensure_ascii=ensure_ascii,
|
| 344 |
+
check_circular=check_circular,
|
| 345 |
+
allow_nan=allow_nan,
|
| 346 |
+
cls=cls,
|
| 347 |
+
indent=indent,
|
| 348 |
+
separators=separators,
|
| 349 |
+
default=default,
|
| 350 |
+
sort_keys=sort_keys,
|
| 351 |
+
**kwargs,
|
| 352 |
+
)
|
|
@@ -0,0 +1,904 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.4.1"
|
| 3 |
+
# deps = []
|
| 4 |
+
# tier = "medium"
|
| 5 |
+
# category = "text"
|
| 6 |
+
# note = "Install/update via `zerodep add markdown`"
|
| 7 |
+
# ///
|
| 8 |
+
|
| 9 |
+
"""Markdown to HTML renderer — zero dependencies, stdlib only, Python 3.10+.
|
| 10 |
+
|
| 11 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 12 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 13 |
+
|
| 14 |
+
Drop-in replacement for mistune's ``mistune.html()`` for common Markdown.
|
| 15 |
+
|
| 16 |
+
Supports:
|
| 17 |
+
- ATX and Setext headings
|
| 18 |
+
- Paragraphs, thematic breaks, hard line breaks
|
| 19 |
+
- Emphasis (bold, italic, bold-italic)
|
| 20 |
+
- Inline code and fenced/indented code blocks
|
| 21 |
+
- Links (inline, reference, autolink) and images
|
| 22 |
+
- Ordered and unordered lists with nesting
|
| 23 |
+
- Block quotes with nesting
|
| 24 |
+
- GFM tables with column alignment
|
| 25 |
+
- GFM strikethrough (~~text~~)
|
| 26 |
+
- GFM task lists (- [ ] / - [x])
|
| 27 |
+
- GFM extended autolinks (bare URLs)
|
| 28 |
+
- Backslash escapes
|
| 29 |
+
|
| 30 |
+
Does NOT implement:
|
| 31 |
+
- Raw HTML passthrough (escaped for safety)
|
| 32 |
+
- Footnotes, definition lists, math/LaTeX
|
| 33 |
+
|
| 34 |
+
Example::
|
| 35 |
+
|
| 36 |
+
from markdown import render
|
| 37 |
+
render("# Hello\\n\\nThis is **bold**.")
|
| 38 |
+
# '<h1>Hello</h1>\\n<p>This is <strong>bold</strong>.</p>\\n'
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
from __future__ import annotations
|
| 42 |
+
|
| 43 |
+
import html
|
| 44 |
+
import re
|
| 45 |
+
|
| 46 |
+
__all__ = [
|
| 47 |
+
"render",
|
| 48 |
+
]
|
| 49 |
+
|
| 50 |
+
# ── Constants ─────────────────────────────────────────────────────────────────
|
| 51 |
+
|
| 52 |
+
_ESCAPED_CHARS = r"\\!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~"
|
| 53 |
+
|
| 54 |
+
_HARMFUL_PROTOCOLS = ("javascript:", "vbscript:", "file:", "data:")
|
| 55 |
+
_SAFE_DATA_PREFIXES = (
|
| 56 |
+
"data:image/gif;",
|
| 57 |
+
"data:image/png;",
|
| 58 |
+
"data:image/jpeg;",
|
| 59 |
+
"data:image/webp;",
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
# ── Block-level patterns ─────────────────────────────────────────────────────
|
| 63 |
+
|
| 64 |
+
_ATX_HEADING_RE = re.compile(r"^(#{1,6})(?:\s+|$)(.*?)(?:\s+#+\s*)?$")
|
| 65 |
+
_SETEXT_HEADING_RE = re.compile(r"^(=+|-+)\s*$")
|
| 66 |
+
_FENCED_CODE_RE = re.compile(r"^( {0,3})(`{3,}|~{3,})\s*(\S*)\s*$")
|
| 67 |
+
_THEMATIC_BREAK_RE = re.compile(
|
| 68 |
+
r"^ {0,3}(?:(?:-[ \t]*){3,}|(?:\*[ \t]*){3,}|(?:_[ \t]*){3,})\s*$"
|
| 69 |
+
)
|
| 70 |
+
_BLOCK_QUOTE_RE = re.compile(r"^ {0,3}>[ \t]?(.*)")
|
| 71 |
+
_UL_ITEM_RE = re.compile(r"^( *)([-*+]) (.*)")
|
| 72 |
+
_OL_ITEM_RE = re.compile(r"^( *)(\d{1,9})([.)]) (.*)")
|
| 73 |
+
_INDENT_CODE_RE = re.compile(r"^(?: {4}|\t)(.*)")
|
| 74 |
+
_REF_LINK_RE = re.compile(
|
| 75 |
+
r"""^ {0,3}\[([^\]]+)\]:\s+<?(\S+?)>?(?:\s+["'(](.+?)["')])?\s*$"""
|
| 76 |
+
)
|
| 77 |
+
_TABLE_DELIM_RE = re.compile(
|
| 78 |
+
r"^ {0,3}\|?[ \t]*:?-+:?[ \t]*(?:\|[ \t]*:?-+:?[ \t]*)*\|?\s*$"
|
| 79 |
+
)
|
| 80 |
+
_BLANK_RE = re.compile(r"^\s*$")
|
| 81 |
+
|
| 82 |
+
# ── Inline patterns ──────────────────────────────────────────────────────────
|
| 83 |
+
|
| 84 |
+
_BACKSLASH_ESCAPE_RE = re.compile(r"\\([" + _ESCAPED_CHARS + r"])")
|
| 85 |
+
_CODE_SPAN_RE = re.compile(r"(?<!`)(`+)(?!`)([\s\S]*?[^`])(\1)(?!`)")
|
| 86 |
+
_AUTOLINK_RE = re.compile(r"<([a-zA-Z][a-zA-Z0-9.+-]{1,31}:[^<>\s]*)>")
|
| 87 |
+
_AUTOEMAIL_RE = re.compile(
|
| 88 |
+
r"<([a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9]"
|
| 89 |
+
r"(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?"
|
| 90 |
+
r"(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)>"
|
| 91 |
+
)
|
| 92 |
+
_IMAGE_RE = re.compile(
|
| 93 |
+
r"!\[([^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*)\]"
|
| 94 |
+
r"""\(\s*(\S+?)(?:\s+["'](.+?)["'])?\s*\)"""
|
| 95 |
+
)
|
| 96 |
+
_LINK_RE = re.compile(
|
| 97 |
+
r"\[([^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*)\]"
|
| 98 |
+
r"""\(\s*(\S+?)(?:\s+["'](.+?)["'])?\s*\)"""
|
| 99 |
+
)
|
| 100 |
+
_REF_LINK_USE_RE = re.compile(r"\[([^\[\]]+)\]\[([^\[\]]*)\]")
|
| 101 |
+
_REF_LINK_SHORT_RE = re.compile(r"\[([^\[\]]+)\](?!\()")
|
| 102 |
+
_STRONG_EM_RE = re.compile(r"\*{3}(.+?)\*{3}|_{3}(.+?)_{3}")
|
| 103 |
+
_STRONG_RE = re.compile(r"\*{2}(.+?)\*{2}|_{2}(.+?)_{2}")
|
| 104 |
+
_EM_RE = re.compile(r"(?<!\w)\*(.+?)\*(?!\w)|(?<!\w)_(.+?)_(?!\w)")
|
| 105 |
+
_STRIKETHROUGH_RE = re.compile(r"~~(.+?)~~")
|
| 106 |
+
_BARE_URL_RE = re.compile(r"(https?://[^\s<>\[\]]*[^\s<>\[\].,;:!?'\")}\]])")
|
| 107 |
+
_HARD_BREAK_RE = re.compile(r"(?:\\| {2,})\n")
|
| 108 |
+
|
| 109 |
+
# Task list marker: [ ], [x], or [X] at start of list item content
|
| 110 |
+
_TASK_LIST_RE = re.compile(r"^\[([ xX])\]\s+")
|
| 111 |
+
|
| 112 |
+
# ── Placeholder system ───────────────────────────────────────────────────────
|
| 113 |
+
|
| 114 |
+
_PH_PREFIX = "\x00PH"
|
| 115 |
+
_PH_SUFFIX = "\x00"
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
class _Placeholders:
|
| 119 |
+
"""Manages placeholder substitution to protect parsed regions."""
|
| 120 |
+
|
| 121 |
+
__slots__ = ("_store", "_counter")
|
| 122 |
+
|
| 123 |
+
def __init__(self) -> None:
|
| 124 |
+
self._store: list[str] = []
|
| 125 |
+
self._counter = 0
|
| 126 |
+
|
| 127 |
+
def put(self, value: str) -> str:
|
| 128 |
+
idx = self._counter
|
| 129 |
+
self._counter += 1
|
| 130 |
+
self._store.append(value)
|
| 131 |
+
return f"{_PH_PREFIX}{idx}{_PH_SUFFIX}"
|
| 132 |
+
|
| 133 |
+
def restore(self, text: str) -> str:
|
| 134 |
+
for i in range(len(self._store) - 1, -1, -1):
|
| 135 |
+
text = text.replace(f"{_PH_PREFIX}{i}{_PH_SUFFIX}", self._store[i])
|
| 136 |
+
return text
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
# ── URL safety ────────��───────────────────────────────────────────────────────
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def _safe_url(url: str) -> str:
|
| 143 |
+
"""Sanitize a URL, blocking dangerous protocols."""
|
| 144 |
+
lower = url.strip().lower()
|
| 145 |
+
if lower.startswith(_HARMFUL_PROTOCOLS) and not lower.startswith(
|
| 146 |
+
_SAFE_DATA_PREFIXES
|
| 147 |
+
):
|
| 148 |
+
return "#harmful-link"
|
| 149 |
+
return html.escape(url, quote=True)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
# ── Inline helper functions ──────────────────────────────────────────────────
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _apply_bare_urls(text: str, ph: _Placeholders) -> str:
|
| 156 |
+
"""Replace bare http/https URLs with link placeholders."""
|
| 157 |
+
|
| 158 |
+
def _bare_url_repl(m: re.Match[str]) -> str:
|
| 159 |
+
url = m.group(1)
|
| 160 |
+
return ph.put('<a href="' + _safe_url(url) + '">' + html.escape(url) + "</a>")
|
| 161 |
+
|
| 162 |
+
return _BARE_URL_RE.sub(_bare_url_repl, text)
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _apply_emphasis(
|
| 166 |
+
text: str,
|
| 167 |
+
ph: _Placeholders,
|
| 168 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 169 |
+
) -> str:
|
| 170 |
+
"""Apply bold-italic, bold, italic, and strikethrough replacements."""
|
| 171 |
+
|
| 172 |
+
# Bold-italic ***text***
|
| 173 |
+
def _strong_em_repl(m: re.Match[str]) -> str:
|
| 174 |
+
content = m.group(1) or m.group(2)
|
| 175 |
+
return ph.put(
|
| 176 |
+
"<em><strong>" + _parse_inline(content, ref_links) + "</strong></em>"
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
text = _STRONG_EM_RE.sub(_strong_em_repl, text)
|
| 180 |
+
|
| 181 |
+
# Bold **text**
|
| 182 |
+
def _strong_repl(m: re.Match[str]) -> str:
|
| 183 |
+
content = m.group(1) or m.group(2)
|
| 184 |
+
return ph.put("<strong>" + _parse_inline(content, ref_links) + "</strong>")
|
| 185 |
+
|
| 186 |
+
text = _STRONG_RE.sub(_strong_repl, text)
|
| 187 |
+
|
| 188 |
+
# Italic *text*
|
| 189 |
+
def _em_repl(m: re.Match[str]) -> str:
|
| 190 |
+
content = m.group(1) or m.group(2)
|
| 191 |
+
return ph.put("<em>" + _parse_inline(content, ref_links) + "</em>")
|
| 192 |
+
|
| 193 |
+
text = _EM_RE.sub(_em_repl, text)
|
| 194 |
+
|
| 195 |
+
# Strikethrough ~~text~~
|
| 196 |
+
def _strike_repl(m: re.Match[str]) -> str:
|
| 197 |
+
content = m.group(1)
|
| 198 |
+
return ph.put("<del>" + _parse_inline(content, ref_links) + "</del>")
|
| 199 |
+
|
| 200 |
+
text = _STRIKETHROUGH_RE.sub(_strike_repl, text)
|
| 201 |
+
return text
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
# ── Inline parser ─────────────────────────────────────────────────────────────
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def _parse_inline(
|
| 208 |
+
text: str,
|
| 209 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 210 |
+
) -> str:
|
| 211 |
+
"""Parse inline Markdown elements and return HTML."""
|
| 212 |
+
ph = _Placeholders()
|
| 213 |
+
|
| 214 |
+
# 1. Backslash escapes → placeholder
|
| 215 |
+
def _escape_repl(m: re.Match[str]) -> str:
|
| 216 |
+
return ph.put(html.escape(m.group(1), quote=True))
|
| 217 |
+
|
| 218 |
+
text = _BACKSLASH_ESCAPE_RE.sub(_escape_repl, text)
|
| 219 |
+
|
| 220 |
+
# 2. Code spans → placeholder (no further parsing inside)
|
| 221 |
+
def _code_repl(m: re.Match[str]) -> str:
|
| 222 |
+
code = m.group(2).strip()
|
| 223 |
+
return ph.put("<code>" + html.escape(code, quote=True) + "</code>")
|
| 224 |
+
|
| 225 |
+
text = _CODE_SPAN_RE.sub(_code_repl, text)
|
| 226 |
+
|
| 227 |
+
# 3. Autolinks
|
| 228 |
+
def _autolink_repl(m: re.Match[str]) -> str:
|
| 229 |
+
url = m.group(1)
|
| 230 |
+
return ph.put('<a href="' + _safe_url(url) + '">' + html.escape(url) + "</a>")
|
| 231 |
+
|
| 232 |
+
text = _AUTOLINK_RE.sub(_autolink_repl, text)
|
| 233 |
+
|
| 234 |
+
# 3b. Auto emails
|
| 235 |
+
def _autoemail_repl(m: re.Match[str]) -> str:
|
| 236 |
+
email = m.group(1)
|
| 237 |
+
return ph.put(
|
| 238 |
+
'<a href="mailto:' + html.escape(email) + '">' + html.escape(email) + "</a>"
|
| 239 |
+
)
|
| 240 |
+
|
| 241 |
+
text = _AUTOEMAIL_RE.sub(_autoemail_repl, text)
|
| 242 |
+
|
| 243 |
+
# 4. Images
|
| 244 |
+
def _image_repl(m: re.Match[str]) -> str:
|
| 245 |
+
alt = html.escape(m.group(1), quote=True)
|
| 246 |
+
url = _safe_url(m.group(2))
|
| 247 |
+
title = m.group(3)
|
| 248 |
+
s = '<img src="' + url + '" alt="' + alt + '"'
|
| 249 |
+
if title:
|
| 250 |
+
s += ' title="' + html.escape(title, quote=True) + '"'
|
| 251 |
+
s += " />"
|
| 252 |
+
return ph.put(s)
|
| 253 |
+
|
| 254 |
+
text = _IMAGE_RE.sub(_image_repl, text)
|
| 255 |
+
|
| 256 |
+
# 5. Inline links
|
| 257 |
+
def _link_repl(m: re.Match[str]) -> str:
|
| 258 |
+
link_text = _parse_inline(m.group(1), ref_links)
|
| 259 |
+
url = _safe_url(m.group(2))
|
| 260 |
+
title = m.group(3)
|
| 261 |
+
s = '<a href="' + url + '"'
|
| 262 |
+
if title:
|
| 263 |
+
s += ' title="' + html.escape(title, quote=True) + '"'
|
| 264 |
+
s += ">" + link_text + "</a>"
|
| 265 |
+
return ph.put(s)
|
| 266 |
+
|
| 267 |
+
text = _LINK_RE.sub(_link_repl, text)
|
| 268 |
+
|
| 269 |
+
# 5b. Reference links [text][ref]
|
| 270 |
+
def _ref_link_repl(m: re.Match[str]) -> str:
|
| 271 |
+
link_text = m.group(1)
|
| 272 |
+
ref_key = (m.group(2) or link_text).strip().lower()
|
| 273 |
+
ref = ref_links.get(ref_key)
|
| 274 |
+
if ref is None:
|
| 275 |
+
return m.group(0)
|
| 276 |
+
url, title = ref
|
| 277 |
+
s = '<a href="' + _safe_url(url) + '"'
|
| 278 |
+
if title:
|
| 279 |
+
s += ' title="' + html.escape(title, quote=True) + '"'
|
| 280 |
+
s += ">" + _parse_inline(link_text, ref_links) + "</a>"
|
| 281 |
+
return ph.put(s)
|
| 282 |
+
|
| 283 |
+
text = _REF_LINK_USE_RE.sub(_ref_link_repl, text)
|
| 284 |
+
|
| 285 |
+
# 5c. Shortcut reference links [ref]
|
| 286 |
+
def _ref_short_repl(m: re.Match[str]) -> str:
|
| 287 |
+
link_text = m.group(1)
|
| 288 |
+
ref_key = link_text.strip().lower()
|
| 289 |
+
ref = ref_links.get(ref_key)
|
| 290 |
+
if ref is None:
|
| 291 |
+
return m.group(0)
|
| 292 |
+
url, title = ref
|
| 293 |
+
s = '<a href="' + _safe_url(url) + '"'
|
| 294 |
+
if title:
|
| 295 |
+
s += ' title="' + html.escape(title, quote=True) + '"'
|
| 296 |
+
s += ">" + _parse_inline(link_text, ref_links) + "</a>"
|
| 297 |
+
return ph.put(s)
|
| 298 |
+
|
| 299 |
+
text = _REF_LINK_SHORT_RE.sub(_ref_short_repl, text)
|
| 300 |
+
|
| 301 |
+
# 5d. Extended autolinks (bare URLs without angle brackets)
|
| 302 |
+
text = _apply_bare_urls(text, ph)
|
| 303 |
+
|
| 304 |
+
# 6. Emphasis + strikethrough
|
| 305 |
+
text = _apply_emphasis(text, ph, ref_links)
|
| 306 |
+
|
| 307 |
+
# 7. Hard line breaks (protect with placeholder)
|
| 308 |
+
text = _HARD_BREAK_RE.sub(lambda m: ph.put("<br />\n"), text)
|
| 309 |
+
|
| 310 |
+
# 8. Escape remaining HTML in text
|
| 311 |
+
# We need to be careful: only escape text that hasn't been processed.
|
| 312 |
+
# The placeholder system handles this — placeholders contain final HTML.
|
| 313 |
+
# We escape the remaining raw text segments between placeholders.
|
| 314 |
+
parts = re.split(r"(\x00PH\d+\x00)", text)
|
| 315 |
+
for i, part in enumerate(parts):
|
| 316 |
+
if not part.startswith(_PH_PREFIX):
|
| 317 |
+
parts[i] = html.escape(part, quote=False)
|
| 318 |
+
text = "".join(parts)
|
| 319 |
+
|
| 320 |
+
# 9. Restore placeholders
|
| 321 |
+
text = ph.restore(text)
|
| 322 |
+
|
| 323 |
+
return text
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
# ── Table parser ──────────────────────────────────────────────────────────────
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def _parse_table_row(line: str) -> list[str]:
|
| 330 |
+
"""Split a table row into cells."""
|
| 331 |
+
line = line.strip()
|
| 332 |
+
if line.startswith("|"):
|
| 333 |
+
line = line[1:]
|
| 334 |
+
if line.endswith("|"):
|
| 335 |
+
line = line[:-1]
|
| 336 |
+
# Split on unescaped pipes
|
| 337 |
+
cells = re.split(r"(?<!\\)\|", line)
|
| 338 |
+
return [c.strip() for c in cells]
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
def _parse_table_align(line: str) -> list[str | None]:
|
| 342 |
+
"""Parse alignment row, return list of 'left', 'right', 'center', or None."""
|
| 343 |
+
cells = _parse_table_row(line)
|
| 344 |
+
aligns: list[str | None] = []
|
| 345 |
+
for cell in cells:
|
| 346 |
+
cell = cell.strip()
|
| 347 |
+
left = cell.startswith(":")
|
| 348 |
+
right = cell.endswith(":")
|
| 349 |
+
if left and right:
|
| 350 |
+
aligns.append("center")
|
| 351 |
+
elif right:
|
| 352 |
+
aligns.append("right")
|
| 353 |
+
elif left:
|
| 354 |
+
aligns.append("left")
|
| 355 |
+
else:
|
| 356 |
+
aligns.append(None)
|
| 357 |
+
return aligns
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def _render_table(
|
| 361 |
+
header_line: str,
|
| 362 |
+
align_line: str,
|
| 363 |
+
body_lines: list[str],
|
| 364 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 365 |
+
) -> str:
|
| 366 |
+
"""Render a GFM table to HTML."""
|
| 367 |
+
headers = _parse_table_row(header_line)
|
| 368 |
+
aligns = _parse_table_align(align_line)
|
| 369 |
+
|
| 370 |
+
out: list[str] = ["<table>\n<thead>\n<tr>\n"]
|
| 371 |
+
for i, h in enumerate(headers):
|
| 372 |
+
align = aligns[i] if i < len(aligns) else None
|
| 373 |
+
style = f' style="text-align:{align}"' if align else ""
|
| 374 |
+
out.append(f" <th{style}>{_parse_inline(h, ref_links)}</th>\n")
|
| 375 |
+
out.append("</tr>\n</thead>\n")
|
| 376 |
+
|
| 377 |
+
if body_lines:
|
| 378 |
+
out.append("<tbody>\n")
|
| 379 |
+
for line in body_lines:
|
| 380 |
+
cells = _parse_table_row(line)
|
| 381 |
+
out.append("<tr>\n")
|
| 382 |
+
for i, cell in enumerate(cells):
|
| 383 |
+
align = aligns[i] if i < len(aligns) else None
|
| 384 |
+
style = f' style="text-align:{align}"' if align else ""
|
| 385 |
+
out.append(f" <td{style}>{_parse_inline(cell, ref_links)}</td>\n")
|
| 386 |
+
out.append("</tr>\n")
|
| 387 |
+
out.append("</tbody>\n")
|
| 388 |
+
|
| 389 |
+
out.append("</table>\n")
|
| 390 |
+
return "".join(out)
|
| 391 |
+
|
| 392 |
+
|
| 393 |
+
# ── List parser ───────────────────────────────────────────────────────────────
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
def _detect_list_type(
|
| 397 |
+
line: str,
|
| 398 |
+
) -> tuple[bool, int, int] | None:
|
| 399 |
+
"""Detect whether a line starts a list item.
|
| 400 |
+
|
| 401 |
+
Returns:
|
| 402 |
+
(ordered, base_indent, start_num) or None if not a list item.
|
| 403 |
+
"""
|
| 404 |
+
ol_m = _OL_ITEM_RE.match(line)
|
| 405 |
+
if ol_m:
|
| 406 |
+
return True, len(ol_m.group(1)), int(ol_m.group(2))
|
| 407 |
+
ul_m = _UL_ITEM_RE.match(line)
|
| 408 |
+
if ul_m:
|
| 409 |
+
return False, len(ul_m.group(1)), 1
|
| 410 |
+
return None
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
def _blank_continues_list(
|
| 414 |
+
lines: list[str],
|
| 415 |
+
idx: int,
|
| 416 |
+
ordered: bool,
|
| 417 |
+
base_indent: int,
|
| 418 |
+
) -> bool:
|
| 419 |
+
"""Check whether a blank line at *idx* is followed by list continuation."""
|
| 420 |
+
if idx + 1 >= len(lines):
|
| 421 |
+
return False
|
| 422 |
+
next_line = lines[idx + 1]
|
| 423 |
+
if _BLANK_RE.match(next_line):
|
| 424 |
+
return False
|
| 425 |
+
# Same-level item?
|
| 426 |
+
if ordered:
|
| 427 |
+
m = _OL_ITEM_RE.match(next_line)
|
| 428 |
+
if m and len(m.group(1)) == base_indent:
|
| 429 |
+
return True
|
| 430 |
+
else:
|
| 431 |
+
m = _UL_ITEM_RE.match(next_line)
|
| 432 |
+
if m and len(m.group(1)) == base_indent:
|
| 433 |
+
return True
|
| 434 |
+
# Indented continuation / nested content
|
| 435 |
+
next_indent = len(next_line) - len(next_line.lstrip())
|
| 436 |
+
return next_indent > base_indent
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def _match_same_level_item(
|
| 440 |
+
line: str,
|
| 441 |
+
ordered: bool,
|
| 442 |
+
base_indent: int,
|
| 443 |
+
) -> str | None:
|
| 444 |
+
"""If *line* is a same-level list item, return its content text."""
|
| 445 |
+
if ordered:
|
| 446 |
+
m = _OL_ITEM_RE.match(line)
|
| 447 |
+
if m and len(m.group(1)) == base_indent:
|
| 448 |
+
return m.group(4)
|
| 449 |
+
else:
|
| 450 |
+
m = _UL_ITEM_RE.match(line)
|
| 451 |
+
if m and len(m.group(1)) == base_indent:
|
| 452 |
+
return m.group(3)
|
| 453 |
+
return None
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
def _collect_list_items(
|
| 457 |
+
lines: list[str],
|
| 458 |
+
start_idx: int,
|
| 459 |
+
ordered: bool,
|
| 460 |
+
base_indent: int,
|
| 461 |
+
) -> tuple[list[list[str]], int]:
|
| 462 |
+
"""Collect raw item line groups from a list block.
|
| 463 |
+
|
| 464 |
+
Returns:
|
| 465 |
+
(items, next_idx) where each item is a list of content lines.
|
| 466 |
+
"""
|
| 467 |
+
idx = start_idx
|
| 468 |
+
items: list[list[str]] = []
|
| 469 |
+
current: list[str] = []
|
| 470 |
+
|
| 471 |
+
while idx < len(lines):
|
| 472 |
+
line = lines[idx]
|
| 473 |
+
|
| 474 |
+
# Blank line — may separate items or end the list
|
| 475 |
+
if _BLANK_RE.match(line):
|
| 476 |
+
if _blank_continues_list(lines, idx, ordered, base_indent):
|
| 477 |
+
current.append("")
|
| 478 |
+
idx += 1
|
| 479 |
+
continue
|
| 480 |
+
break
|
| 481 |
+
|
| 482 |
+
# Same-level item starts a new entry
|
| 483 |
+
content = _match_same_level_item(line, ordered, base_indent)
|
| 484 |
+
if content is not None:
|
| 485 |
+
if current:
|
| 486 |
+
items.append(current)
|
| 487 |
+
current = [content]
|
| 488 |
+
idx += 1
|
| 489 |
+
continue
|
| 490 |
+
|
| 491 |
+
# Nested / continuation line
|
| 492 |
+
indent = len(line) - len(line.lstrip())
|
| 493 |
+
if indent > base_indent and current:
|
| 494 |
+
dedented = line[base_indent + 2 :] if len(line) > base_indent + 2 else ""
|
| 495 |
+
current.append(dedented)
|
| 496 |
+
idx += 1
|
| 497 |
+
continue
|
| 498 |
+
|
| 499 |
+
break
|
| 500 |
+
|
| 501 |
+
if current:
|
| 502 |
+
items.append(current)
|
| 503 |
+
return items, idx
|
| 504 |
+
|
| 505 |
+
|
| 506 |
+
def _render_list_item(
|
| 507 |
+
item_lines: list[str],
|
| 508 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 509 |
+
) -> str:
|
| 510 |
+
"""Render a single <li> element, recursing for nested sub-lists."""
|
| 511 |
+
# Detect task list marker
|
| 512 |
+
task_class = ""
|
| 513 |
+
task_checkbox = ""
|
| 514 |
+
first_line = item_lines[0]
|
| 515 |
+
task_m = _TASK_LIST_RE.match(first_line)
|
| 516 |
+
if task_m:
|
| 517 |
+
checked = task_m.group(1) in ("x", "X")
|
| 518 |
+
task_class = ' class="task-list-item"'
|
| 519 |
+
task_checkbox = (
|
| 520 |
+
'<input class="task-list-item-checkbox" type="checkbox" disabled'
|
| 521 |
+
)
|
| 522 |
+
if checked:
|
| 523 |
+
task_checkbox += " checked"
|
| 524 |
+
task_checkbox += "/>"
|
| 525 |
+
item_lines = [first_line[task_m.end() :]] + item_lines[1:]
|
| 526 |
+
|
| 527 |
+
has_sublist = any(
|
| 528 |
+
_UL_ITEM_RE.match(il) or _OL_ITEM_RE.match(il) for il in item_lines[1:]
|
| 529 |
+
)
|
| 530 |
+
open_tag = f"<li{task_class}>" + task_checkbox
|
| 531 |
+
if has_sublist:
|
| 532 |
+
first = _parse_inline(item_lines[0], ref_links)
|
| 533 |
+
sub_html = _parse_blocks(item_lines[1:], ref_links)
|
| 534 |
+
return open_tag + first + sub_html + "</li>\n"
|
| 535 |
+
content = "\n".join(item_lines).strip()
|
| 536 |
+
return open_tag + _parse_inline(content, ref_links) + "</li>\n"
|
| 537 |
+
|
| 538 |
+
|
| 539 |
+
def _parse_list_block(
|
| 540 |
+
lines: list[str],
|
| 541 |
+
start_idx: int,
|
| 542 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 543 |
+
) -> tuple[str, int]:
|
| 544 |
+
"""Parse a list block starting at *start_idx*. Returns (html, next_idx)."""
|
| 545 |
+
info = _detect_list_type(lines[start_idx])
|
| 546 |
+
if info is None:
|
| 547 |
+
return "", start_idx
|
| 548 |
+
ordered, base_indent, start_num = info
|
| 549 |
+
|
| 550 |
+
items, idx = _collect_list_items(lines, start_idx, ordered, base_indent)
|
| 551 |
+
|
| 552 |
+
tag = "ol" if ordered else "ul"
|
| 553 |
+
start_attr = f' start="{start_num}"' if ordered and start_num != 1 else ""
|
| 554 |
+
parts: list[str] = [f"<{tag}{start_attr}>\n"]
|
| 555 |
+
for item_lines in items:
|
| 556 |
+
parts.append(_render_list_item(item_lines, ref_links))
|
| 557 |
+
parts.append(f"</{tag}>\n")
|
| 558 |
+
return "".join(parts), idx
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
# ── Block-level try-parse helpers ────────────────────────────────────────────
|
| 562 |
+
|
| 563 |
+
_BlockResult = tuple[str, int, bool]
|
| 564 |
+
"""(html, new_idx, consumed_para) returned by each _try_parse_* helper."""
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
def _try_parse_hr(
|
| 568 |
+
lines: list[str],
|
| 569 |
+
idx: int,
|
| 570 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 571 |
+
para_lines: list[str],
|
| 572 |
+
) -> _BlockResult | None:
|
| 573 |
+
"""Try to parse a thematic break."""
|
| 574 |
+
if para_lines:
|
| 575 |
+
return None
|
| 576 |
+
if _THEMATIC_BREAK_RE.match(lines[idx]):
|
| 577 |
+
return "<hr />\n", idx + 1, False
|
| 578 |
+
return None
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
def _try_parse_atx_heading(
|
| 582 |
+
lines: list[str],
|
| 583 |
+
idx: int,
|
| 584 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 585 |
+
para_lines: list[str],
|
| 586 |
+
) -> _BlockResult | None:
|
| 587 |
+
"""Try to parse an ATX heading."""
|
| 588 |
+
if para_lines:
|
| 589 |
+
return None
|
| 590 |
+
m = _ATX_HEADING_RE.match(lines[idx])
|
| 591 |
+
if not m:
|
| 592 |
+
return None
|
| 593 |
+
level = len(m.group(1))
|
| 594 |
+
content = re.sub(r"\s+#+\s*$", "", m.group(2).strip())
|
| 595 |
+
tag = f"h{level}"
|
| 596 |
+
h = f"<{tag}>{_parse_inline(content, ref_links)}</{tag}>\n"
|
| 597 |
+
return h, idx + 1, False
|
| 598 |
+
|
| 599 |
+
|
| 600 |
+
def _try_parse_setext_heading(
|
| 601 |
+
lines: list[str],
|
| 602 |
+
idx: int,
|
| 603 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 604 |
+
para_lines: list[str],
|
| 605 |
+
) -> _BlockResult | None:
|
| 606 |
+
"""Try to parse a Setext heading (requires accumulated paragraph lines)."""
|
| 607 |
+
if not para_lines:
|
| 608 |
+
return None
|
| 609 |
+
line = lines[idx]
|
| 610 |
+
if not _SETEXT_HEADING_RE.match(line):
|
| 611 |
+
return None
|
| 612 |
+
level = 1 if line.strip().startswith("=") else 2
|
| 613 |
+
content = "\n".join(para_lines)
|
| 614 |
+
tag = f"h{level}"
|
| 615 |
+
h = f"<{tag}>{_parse_inline(content.strip(), ref_links)}</{tag}>\n"
|
| 616 |
+
return h, idx + 1, True # consumed_para=True
|
| 617 |
+
|
| 618 |
+
|
| 619 |
+
def _try_parse_code_fence(
|
| 620 |
+
lines: list[str],
|
| 621 |
+
idx: int,
|
| 622 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 623 |
+
para_lines: list[str],
|
| 624 |
+
) -> _BlockResult | None:
|
| 625 |
+
"""Try to parse a fenced code block."""
|
| 626 |
+
if para_lines:
|
| 627 |
+
return None
|
| 628 |
+
m = _FENCED_CODE_RE.match(lines[idx])
|
| 629 |
+
if not m:
|
| 630 |
+
return None
|
| 631 |
+
fence_indent = len(m.group(1))
|
| 632 |
+
fence_char = m.group(2)[0]
|
| 633 |
+
fence_len = len(m.group(2))
|
| 634 |
+
lang = m.group(3).strip()
|
| 635 |
+
|
| 636 |
+
code_lines: list[str] = []
|
| 637 |
+
j = idx + 1
|
| 638 |
+
close_re = re.compile(
|
| 639 |
+
r"^ {0,3}" + re.escape(fence_char) + r"{" + str(fence_len) + r",}\s*$"
|
| 640 |
+
)
|
| 641 |
+
while j < len(lines):
|
| 642 |
+
cl = lines[j]
|
| 643 |
+
if close_re.match(cl):
|
| 644 |
+
j += 1
|
| 645 |
+
break
|
| 646 |
+
code_lines.append(_strip_fence_indent(cl, fence_indent))
|
| 647 |
+
j += 1
|
| 648 |
+
|
| 649 |
+
code = "\n".join(code_lines)
|
| 650 |
+
if code and not code.endswith("\n"):
|
| 651 |
+
code += "\n"
|
| 652 |
+
escaped_code = html.escape(code, quote=False)
|
| 653 |
+
lang_attr = f' class="language-{html.escape(lang, quote=True)}"' if lang else ""
|
| 654 |
+
h = f"<pre><code{lang_attr}>{escaped_code}</code></pre>\n"
|
| 655 |
+
return h, j, False
|
| 656 |
+
|
| 657 |
+
|
| 658 |
+
def _strip_fence_indent(line: str, indent: int) -> str:
|
| 659 |
+
"""Remove up to *indent* leading spaces from *line*."""
|
| 660 |
+
if indent <= 0:
|
| 661 |
+
return line
|
| 662 |
+
for _ in range(indent):
|
| 663 |
+
if line.startswith(" "):
|
| 664 |
+
line = line[1:]
|
| 665 |
+
else:
|
| 666 |
+
break
|
| 667 |
+
return line
|
| 668 |
+
|
| 669 |
+
|
| 670 |
+
def _try_parse_indent_code(
|
| 671 |
+
lines: list[str],
|
| 672 |
+
idx: int,
|
| 673 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 674 |
+
para_lines: list[str],
|
| 675 |
+
) -> _BlockResult | None:
|
| 676 |
+
"""Try to parse an indented code block (4-space indent)."""
|
| 677 |
+
if para_lines:
|
| 678 |
+
return None
|
| 679 |
+
if not _INDENT_CODE_RE.match(lines[idx]):
|
| 680 |
+
return None
|
| 681 |
+
|
| 682 |
+
code_lines: list[str] = []
|
| 683 |
+
j = idx
|
| 684 |
+
while j < len(lines):
|
| 685 |
+
ic_m = _INDENT_CODE_RE.match(lines[j])
|
| 686 |
+
if ic_m:
|
| 687 |
+
code_lines.append(ic_m.group(1))
|
| 688 |
+
j += 1
|
| 689 |
+
elif _BLANK_RE.match(lines[j]):
|
| 690 |
+
if j + 1 < len(lines) and _INDENT_CODE_RE.match(lines[j + 1]):
|
| 691 |
+
code_lines.append("")
|
| 692 |
+
j += 1
|
| 693 |
+
else:
|
| 694 |
+
break
|
| 695 |
+
else:
|
| 696 |
+
break
|
| 697 |
+
# Remove trailing blank lines
|
| 698 |
+
while code_lines and code_lines[-1] == "":
|
| 699 |
+
code_lines.pop()
|
| 700 |
+
escaped_code = html.escape("\n".join(code_lines), quote=False)
|
| 701 |
+
h = f"<pre><code>{escaped_code}</code></pre>\n"
|
| 702 |
+
return h, j, False
|
| 703 |
+
|
| 704 |
+
|
| 705 |
+
def _try_parse_blockquote(
|
| 706 |
+
lines: list[str],
|
| 707 |
+
idx: int,
|
| 708 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 709 |
+
para_lines: list[str],
|
| 710 |
+
) -> _BlockResult | None:
|
| 711 |
+
"""Try to parse a block quote (can interrupt a paragraph)."""
|
| 712 |
+
if not _BLOCK_QUOTE_RE.match(lines[idx]):
|
| 713 |
+
return None
|
| 714 |
+
|
| 715 |
+
bq_lines: list[str] = []
|
| 716 |
+
j = idx
|
| 717 |
+
while j < len(lines):
|
| 718 |
+
bq_match = _BLOCK_QUOTE_RE.match(lines[j])
|
| 719 |
+
if bq_match:
|
| 720 |
+
bq_lines.append(bq_match.group(1))
|
| 721 |
+
j += 1
|
| 722 |
+
elif _BLANK_RE.match(lines[j]):
|
| 723 |
+
if j + 1 < len(lines) and _BLOCK_QUOTE_RE.match(lines[j + 1]):
|
| 724 |
+
bq_lines.append("")
|
| 725 |
+
j += 1
|
| 726 |
+
else:
|
| 727 |
+
break
|
| 728 |
+
elif lines[j].strip():
|
| 729 |
+
# Lazy continuation
|
| 730 |
+
bq_lines.append(lines[j])
|
| 731 |
+
j += 1
|
| 732 |
+
else:
|
| 733 |
+
break
|
| 734 |
+
inner = _parse_blocks(bq_lines, ref_links)
|
| 735 |
+
h = "<blockquote>\n" + inner + "</blockquote>\n"
|
| 736 |
+
return h, j, False
|
| 737 |
+
|
| 738 |
+
|
| 739 |
+
def _try_parse_list(
|
| 740 |
+
lines: list[str],
|
| 741 |
+
idx: int,
|
| 742 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 743 |
+
para_lines: list[str],
|
| 744 |
+
) -> _BlockResult | None:
|
| 745 |
+
"""Try to parse an ordered or unordered list."""
|
| 746 |
+
if para_lines:
|
| 747 |
+
return None
|
| 748 |
+
if not (_UL_ITEM_RE.match(lines[idx]) or _OL_ITEM_RE.match(lines[idx])):
|
| 749 |
+
return None
|
| 750 |
+
list_html, new_idx = _parse_list_block(lines, idx, ref_links)
|
| 751 |
+
return list_html, new_idx, False
|
| 752 |
+
|
| 753 |
+
|
| 754 |
+
def _try_parse_table(
|
| 755 |
+
lines: list[str],
|
| 756 |
+
idx: int,
|
| 757 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 758 |
+
para_lines: list[str],
|
| 759 |
+
) -> _BlockResult | None:
|
| 760 |
+
"""Try to parse a GFM table."""
|
| 761 |
+
if para_lines:
|
| 762 |
+
return None
|
| 763 |
+
line = lines[idx]
|
| 764 |
+
if "|" not in line or idx + 1 >= len(lines):
|
| 765 |
+
return None
|
| 766 |
+
if not _TABLE_DELIM_RE.match(lines[idx + 1]):
|
| 767 |
+
return None
|
| 768 |
+
|
| 769 |
+
header_line = line
|
| 770 |
+
align_line = lines[idx + 1]
|
| 771 |
+
body: list[str] = []
|
| 772 |
+
j = idx + 2
|
| 773 |
+
while j < len(lines):
|
| 774 |
+
tl = lines[j]
|
| 775 |
+
if "|" in tl and not _BLANK_RE.match(tl):
|
| 776 |
+
body.append(tl)
|
| 777 |
+
j += 1
|
| 778 |
+
else:
|
| 779 |
+
break
|
| 780 |
+
h = _render_table(header_line, align_line, body, ref_links)
|
| 781 |
+
return h, j, False
|
| 782 |
+
|
| 783 |
+
|
| 784 |
+
# Ordered list of try-parse functions used by the block dispatcher.
|
| 785 |
+
_BLOCK_PARSERS = (
|
| 786 |
+
_try_parse_hr,
|
| 787 |
+
_try_parse_atx_heading,
|
| 788 |
+
_try_parse_setext_heading,
|
| 789 |
+
_try_parse_code_fence,
|
| 790 |
+
_try_parse_indent_code,
|
| 791 |
+
_try_parse_blockquote,
|
| 792 |
+
_try_parse_list,
|
| 793 |
+
_try_parse_table,
|
| 794 |
+
)
|
| 795 |
+
|
| 796 |
+
|
| 797 |
+
# ── Block parser ──────────────────────────────────────────────────────────────
|
| 798 |
+
|
| 799 |
+
|
| 800 |
+
def _collect_ref_links(
|
| 801 |
+
lines: list[str],
|
| 802 |
+
) -> dict[str, tuple[str, str | None]]:
|
| 803 |
+
"""First pass: collect reference link definitions."""
|
| 804 |
+
refs: dict[str, tuple[str, str | None]] = {}
|
| 805 |
+
for line in lines:
|
| 806 |
+
m = _REF_LINK_RE.match(line)
|
| 807 |
+
if m:
|
| 808 |
+
key = m.group(1).strip().lower()
|
| 809 |
+
url = m.group(2)
|
| 810 |
+
title = m.group(3)
|
| 811 |
+
refs[key] = (url, title)
|
| 812 |
+
return refs
|
| 813 |
+
|
| 814 |
+
|
| 815 |
+
def _parse_blocks(
|
| 816 |
+
lines: list[str],
|
| 817 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 818 |
+
) -> str:
|
| 819 |
+
"""Parse block-level Markdown elements from a list of lines."""
|
| 820 |
+
out: list[str] = []
|
| 821 |
+
idx = 0
|
| 822 |
+
para_lines: list[str] = []
|
| 823 |
+
|
| 824 |
+
def flush_paragraph() -> None:
|
| 825 |
+
if para_lines:
|
| 826 |
+
text = "\n".join(para_lines)
|
| 827 |
+
out.append("<p>" + _parse_inline(text.strip(), ref_links) + "</p>\n")
|
| 828 |
+
para_lines.clear()
|
| 829 |
+
|
| 830 |
+
while idx < len(lines):
|
| 831 |
+
line = lines[idx]
|
| 832 |
+
|
| 833 |
+
# Skip reference link definitions (already collected)
|
| 834 |
+
if _REF_LINK_RE.match(line):
|
| 835 |
+
flush_paragraph()
|
| 836 |
+
idx += 1
|
| 837 |
+
continue
|
| 838 |
+
|
| 839 |
+
# Blank line
|
| 840 |
+
if _BLANK_RE.match(line):
|
| 841 |
+
flush_paragraph()
|
| 842 |
+
idx += 1
|
| 843 |
+
continue
|
| 844 |
+
|
| 845 |
+
# Try each block-level parser in priority order
|
| 846 |
+
result = _dispatch_block(lines, idx, ref_links, para_lines)
|
| 847 |
+
if result is not None:
|
| 848 |
+
block_html, idx, consumed_para = result
|
| 849 |
+
if consumed_para:
|
| 850 |
+
para_lines.clear()
|
| 851 |
+
else:
|
| 852 |
+
flush_paragraph()
|
| 853 |
+
out.append(block_html)
|
| 854 |
+
continue
|
| 855 |
+
|
| 856 |
+
# Paragraph accumulation
|
| 857 |
+
para_lines.append(line)
|
| 858 |
+
idx += 1
|
| 859 |
+
|
| 860 |
+
flush_paragraph()
|
| 861 |
+
return "".join(out)
|
| 862 |
+
|
| 863 |
+
|
| 864 |
+
def _dispatch_block(
|
| 865 |
+
lines: list[str],
|
| 866 |
+
idx: int,
|
| 867 |
+
ref_links: dict[str, tuple[str, str | None]],
|
| 868 |
+
para_lines: list[str],
|
| 869 |
+
) -> _BlockResult | None:
|
| 870 |
+
"""Try each registered block parser; return first match or None."""
|
| 871 |
+
for try_fn in _BLOCK_PARSERS:
|
| 872 |
+
result = try_fn(lines, idx, ref_links, para_lines)
|
| 873 |
+
if result is not None:
|
| 874 |
+
return result
|
| 875 |
+
return None
|
| 876 |
+
|
| 877 |
+
|
| 878 |
+
# ── Public API ────────────────────────────────────────────────────────────────
|
| 879 |
+
|
| 880 |
+
|
| 881 |
+
def render(text: str) -> str:
|
| 882 |
+
"""Convert Markdown text to HTML.
|
| 883 |
+
|
| 884 |
+
Args:
|
| 885 |
+
text: Markdown source string.
|
| 886 |
+
|
| 887 |
+
Returns:
|
| 888 |
+
HTML string.
|
| 889 |
+
"""
|
| 890 |
+
# Normalize line endings
|
| 891 |
+
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
| 892 |
+
|
| 893 |
+
# Split into lines
|
| 894 |
+
lines = text.split("\n")
|
| 895 |
+
|
| 896 |
+
# Remove trailing newline that produces empty last element
|
| 897 |
+
if lines and lines[-1] == "":
|
| 898 |
+
lines.pop()
|
| 899 |
+
|
| 900 |
+
# First pass: collect reference link definitions
|
| 901 |
+
ref_links = _collect_ref_links(lines)
|
| 902 |
+
|
| 903 |
+
# Second pass: parse blocks
|
| 904 |
+
return _parse_blocks(lines, ref_links)
|
|
@@ -0,0 +1,1002 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.1.0"
|
| 3 |
+
# deps = ["soup"]
|
| 4 |
+
# tier = "medium"
|
| 5 |
+
# category = "text"
|
| 6 |
+
# note = "Install/update via `zerodep add readability`"
|
| 7 |
+
# ///
|
| 8 |
+
|
| 9 |
+
"""HTML readability content extractor — zero-dep, stdlib only, Python 3.10+.
|
| 10 |
+
|
| 11 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 12 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 13 |
+
|
| 14 |
+
Extracts the main article content from arbitrary web pages using a scoring
|
| 15 |
+
algorithm inspired by Mozilla's Readability.js (Firefox Reader View). Built
|
| 16 |
+
on top of ``zerodep/soup`` for HTML parsing; no external dependencies.
|
| 17 |
+
|
| 18 |
+
Algorithm overview:
|
| 19 |
+
|
| 20 |
+
1. Pre-clean the DOM (remove scripts, styles, etc.)
|
| 21 |
+
2. Extract metadata (JSON-LD, ``<meta>`` tags, ``<title>``)
|
| 22 |
+
3. Remove unlikely candidate nodes (sidebars, footers, ads …)
|
| 23 |
+
4. Transform mis-used ``<div>`` elements into ``<p>`` paragraphs
|
| 24 |
+
5. Score every ``<p>``/``<pre>``/``<td>`` node based on comma count,
|
| 25 |
+
text length and class/id weight; propagate scores to parent &
|
| 26 |
+
grandparent
|
| 27 |
+
6. Pick the highest-scoring container and include qualifying siblings
|
| 28 |
+
7. Sanitize the extracted article (remove forms, low-quality headers …)
|
| 29 |
+
8. If the result is too short, retry with relaxed heuristics
|
| 30 |
+
|
| 31 |
+
Example::
|
| 32 |
+
|
| 33 |
+
from readability import extract, is_probably_readable
|
| 34 |
+
|
| 35 |
+
html = open("article.html").read()
|
| 36 |
+
if is_probably_readable(html):
|
| 37 |
+
result = extract(html)
|
| 38 |
+
print(result.title)
|
| 39 |
+
print(result.text[:200])
|
| 40 |
+
|
| 41 |
+
References:
|
| 42 |
+
- Mozilla Readability.js: https://github.com/mozilla/readability
|
| 43 |
+
- python-readability: https://github.com/buriy/python-readability
|
| 44 |
+
"""
|
| 45 |
+
|
| 46 |
+
from __future__ import annotations
|
| 47 |
+
|
| 48 |
+
import json
|
| 49 |
+
import logging
|
| 50 |
+
import math
|
| 51 |
+
import os
|
| 52 |
+
import re
|
| 53 |
+
import sys
|
| 54 |
+
from dataclasses import dataclass
|
| 55 |
+
from html import unescape
|
| 56 |
+
from typing import Any
|
| 57 |
+
|
| 58 |
+
__all__ = [
|
| 59 |
+
"ReadabilityResult",
|
| 60 |
+
"extract",
|
| 61 |
+
"is_probably_readable",
|
| 62 |
+
]
|
| 63 |
+
|
| 64 |
+
log = logging.getLogger(__name__)
|
| 65 |
+
|
| 66 |
+
# ── Lazy soup import ─────────────────────────────────────────────────────────
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _ensure_sibling_path(name: str) -> str:
|
| 70 |
+
"""Add a sibling module directory to ``sys.path`` if not present."""
|
| 71 |
+
sibling_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", name))
|
| 72 |
+
if sibling_dir not in sys.path:
|
| 73 |
+
sys.path.insert(0, sibling_dir)
|
| 74 |
+
return sibling_dir
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _load_soup():
|
| 78 |
+
_ensure_sibling_path("soup")
|
| 79 |
+
try:
|
| 80 |
+
from soup import Soup, Tag # type: ignore[import-untyped]
|
| 81 |
+
except ImportError as exc:
|
| 82 |
+
raise NotImplementedError(
|
| 83 |
+
"readability requires the 'soup' zerodep module — "
|
| 84 |
+
"place soup/soup.py alongside readability/"
|
| 85 |
+
) from exc
|
| 86 |
+
return Soup, Tag
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
# ── Constants & regex patterns ───────────────────────────────────────────────
|
| 90 |
+
|
| 91 |
+
MIN_PARAGRAPH_LENGTH = 25
|
| 92 |
+
RETRY_LENGTH = 250
|
| 93 |
+
DEFAULT_CHAR_THRESHOLD = 500
|
| 94 |
+
|
| 95 |
+
UNLIKELY_CANDIDATES_RE = re.compile(
|
| 96 |
+
r"combx|comment|community|disqus|extra|foot|header|menu|remark|rss|"
|
| 97 |
+
r"shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|"
|
| 98 |
+
r"tweet|twitter|widget|breadcrumb|social|share|related|banner|"
|
| 99 |
+
r"cookie|consent|modal|overlay|nav\b",
|
| 100 |
+
re.I,
|
| 101 |
+
)
|
| 102 |
+
|
| 103 |
+
OK_MAYBE_CANDIDATE_RE = re.compile(
|
| 104 |
+
r"and|article|body|column|main|shadow|content",
|
| 105 |
+
re.I,
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
POSITIVE_RE = re.compile(
|
| 109 |
+
r"article|body|content|entry|hentry|h-entry|main|page|pagination|"
|
| 110 |
+
r"post|text|blog|story",
|
| 111 |
+
re.I,
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
NEGATIVE_RE = re.compile(
|
| 115 |
+
r"-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|"
|
| 116 |
+
r"contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|"
|
| 117 |
+
r"scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|"
|
| 118 |
+
r"tool|widget",
|
| 119 |
+
re.I,
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
BLOCK_LEVEL_TAGS = frozenset(
|
| 123 |
+
{
|
| 124 |
+
"a",
|
| 125 |
+
"blockquote",
|
| 126 |
+
"dl",
|
| 127 |
+
"div",
|
| 128 |
+
"img",
|
| 129 |
+
"ol",
|
| 130 |
+
"p",
|
| 131 |
+
"pre",
|
| 132 |
+
"table",
|
| 133 |
+
"ul",
|
| 134 |
+
"section",
|
| 135 |
+
"figure",
|
| 136 |
+
"header",
|
| 137 |
+
"footer",
|
| 138 |
+
"nav",
|
| 139 |
+
"aside",
|
| 140 |
+
"details",
|
| 141 |
+
"fieldset",
|
| 142 |
+
"form",
|
| 143 |
+
"hr",
|
| 144 |
+
"noscript",
|
| 145 |
+
"video",
|
| 146 |
+
"audio",
|
| 147 |
+
}
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
TAGS_TO_SCORE = frozenset({"p", "pre", "td"})
|
| 151 |
+
|
| 152 |
+
TAG_WEIGHTS: dict[str, int] = {
|
| 153 |
+
"div": 5,
|
| 154 |
+
"article": 5,
|
| 155 |
+
"pre": 3,
|
| 156 |
+
"td": 3,
|
| 157 |
+
"blockquote": 3,
|
| 158 |
+
"address": -3,
|
| 159 |
+
"ol": -3,
|
| 160 |
+
"ul": -3,
|
| 161 |
+
"dl": -3,
|
| 162 |
+
"dd": -3,
|
| 163 |
+
"dt": -3,
|
| 164 |
+
"li": -3,
|
| 165 |
+
"form": -3,
|
| 166 |
+
"aside": -3,
|
| 167 |
+
"h1": -5,
|
| 168 |
+
"h2": -5,
|
| 169 |
+
"h3": -5,
|
| 170 |
+
"h4": -5,
|
| 171 |
+
"h5": -5,
|
| 172 |
+
"h6": -5,
|
| 173 |
+
"th": -5,
|
| 174 |
+
"header": -5,
|
| 175 |
+
"footer": -5,
|
| 176 |
+
"nav": -5,
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
CLEAN_CONDITIONALLY_TAGS = frozenset(
|
| 180 |
+
{"table", "ul", "div", "aside", "header", "footer", "section"}
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
REMOVE_TAGS = frozenset({"form", "textarea", "input", "button", "select"})
|
| 184 |
+
|
| 185 |
+
TITLE_SEPARATORS_RE = re.compile(r"\s+[\|\-–—\\/>»]\s+")
|
| 186 |
+
|
| 187 |
+
# JSON-LD Article types (Schema.org)
|
| 188 |
+
JSONLD_ARTICLE_TYPES = frozenset(
|
| 189 |
+
{
|
| 190 |
+
"Article",
|
| 191 |
+
"AdvertiserContentArticle",
|
| 192 |
+
"NewsArticle",
|
| 193 |
+
"AnalysisNewsArticle",
|
| 194 |
+
"AskPublicNewsArticle",
|
| 195 |
+
"BackgroundNewsArticle",
|
| 196 |
+
"OpinionNewsArticle",
|
| 197 |
+
"ReportageNewsArticle",
|
| 198 |
+
"ReviewNewsArticle",
|
| 199 |
+
"Report",
|
| 200 |
+
"SatiricalArticle",
|
| 201 |
+
"ScholarlyArticle",
|
| 202 |
+
"MedicalScholarlyArticle",
|
| 203 |
+
"SocialMediaPosting",
|
| 204 |
+
"BlogPosting",
|
| 205 |
+
"LiveBlogPosting",
|
| 206 |
+
"DiscussionForumPosting",
|
| 207 |
+
"TechArticle",
|
| 208 |
+
"APIReference",
|
| 209 |
+
}
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
# Multi-language commas for scoring
|
| 213 |
+
COMMAS_RE = re.compile(r"[\u002C\u060C\uFE50\uFE10\uFE11\u2E41\u2E34\u2E32\uFF0C]")
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
# ── Result dataclass ─────────────────────────────────────────────────────────
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
@dataclass(frozen=True)
|
| 220 |
+
class ReadabilityResult:
|
| 221 |
+
"""Container for extracted article data.
|
| 222 |
+
|
| 223 |
+
Attributes:
|
| 224 |
+
title: Article title (refined from ``<title>`` or headings).
|
| 225 |
+
content: Cleaned HTML of the main content.
|
| 226 |
+
text: Plain-text rendering of the main content.
|
| 227 |
+
author: Author name, or ``None``.
|
| 228 |
+
excerpt: Article excerpt / description, or ``None``.
|
| 229 |
+
site_name: Site name (e.g. from ``og:site_name``), or ``None``.
|
| 230 |
+
published_time: Publication timestamp string, or ``None``.
|
| 231 |
+
lang: Language code from ``<html lang="...">``, or ``None``.
|
| 232 |
+
dir: Text direction (``"ltr"`` / ``"rtl"``), or ``None``.
|
| 233 |
+
length: Character count of *text*.
|
| 234 |
+
"""
|
| 235 |
+
|
| 236 |
+
title: str
|
| 237 |
+
content: str
|
| 238 |
+
text: str
|
| 239 |
+
author: str | None = None
|
| 240 |
+
excerpt: str | None = None
|
| 241 |
+
site_name: str | None = None
|
| 242 |
+
published_time: str | None = None
|
| 243 |
+
lang: str | None = None
|
| 244 |
+
dir: str | None = None
|
| 245 |
+
length: int = 0
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
# ── Public API ───────────────────────────────────────────────────────────────
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def extract(html: str, url: str | None = None) -> ReadabilityResult:
|
| 252 |
+
"""Extract the main article content from an HTML string.
|
| 253 |
+
|
| 254 |
+
Args:
|
| 255 |
+
html: The full HTML document as a decoded string.
|
| 256 |
+
url: Optional base URL (currently unused; reserved for future
|
| 257 |
+
link absolutisation).
|
| 258 |
+
|
| 259 |
+
Returns:
|
| 260 |
+
A ``ReadabilityResult`` with the extracted content and metadata.
|
| 261 |
+
"""
|
| 262 |
+
Soup, _Tag = _load_soup()
|
| 263 |
+
reader = _Readability(html, Soup, _Tag)
|
| 264 |
+
return reader.parse()
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def is_probably_readable(
|
| 268 |
+
html: str,
|
| 269 |
+
min_score: float = 20.0,
|
| 270 |
+
min_content_length: int = 140,
|
| 271 |
+
) -> bool:
|
| 272 |
+
"""Quick heuristic check whether *html* likely contains a readable article.
|
| 273 |
+
|
| 274 |
+
Uses the same approach as Mozilla's ``isProbablyReaderable``: accumulate
|
| 275 |
+
``sqrt(textLen - threshold)`` over qualifying ``<p>``/``<pre>``/``<article>``
|
| 276 |
+
nodes and return ``True`` once the score exceeds *min_score*.
|
| 277 |
+
|
| 278 |
+
Args:
|
| 279 |
+
html: The HTML document string.
|
| 280 |
+
min_score: Minimum cumulative score to consider readable.
|
| 281 |
+
min_content_length: Minimum text length for a node to contribute.
|
| 282 |
+
|
| 283 |
+
Returns:
|
| 284 |
+
``True`` if the page is probably an article.
|
| 285 |
+
"""
|
| 286 |
+
Soup, _Tag = _load_soup()
|
| 287 |
+
soup = Soup(html)
|
| 288 |
+
|
| 289 |
+
score = 0.0
|
| 290 |
+
for tag in soup.find_all(["p", "pre", "article"]):
|
| 291 |
+
# Skip unlikely candidates
|
| 292 |
+
class_id = _get_class_id_string(tag)
|
| 293 |
+
if len(class_id) > 1:
|
| 294 |
+
is_unlikely = UNLIKELY_CANDIDATES_RE.search(class_id)
|
| 295 |
+
is_ok = OK_MAYBE_CANDIDATE_RE.search(class_id)
|
| 296 |
+
if is_unlikely and not is_ok:
|
| 297 |
+
continue
|
| 298 |
+
|
| 299 |
+
text = tag.get_text(strip=True)
|
| 300 |
+
text_len = len(text)
|
| 301 |
+
if text_len < min_content_length:
|
| 302 |
+
continue
|
| 303 |
+
|
| 304 |
+
score += math.sqrt(text_len - min_content_length)
|
| 305 |
+
if score >= min_score:
|
| 306 |
+
return True
|
| 307 |
+
|
| 308 |
+
return False
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
# ── Internal helpers ─────────────────────────────────────────────────────────
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def _get_class_id_string(tag: Any) -> str:
|
| 315 |
+
"""Return concatenated class and id for regex matching."""
|
| 316 |
+
cls = tag.get("class", [])
|
| 317 |
+
if isinstance(cls, list):
|
| 318 |
+
cls = " ".join(cls)
|
| 319 |
+
tag_id = tag.get("id", "")
|
| 320 |
+
return f"{cls} {tag_id}"
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
def _normalize_spaces(s: str) -> str:
|
| 324 |
+
"""Collapse all whitespace to single spaces and strip."""
|
| 325 |
+
return " ".join(s.split())
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def _text_length(tag: Any) -> int:
|
| 329 |
+
"""Return the length of the whitespace-normalised text content."""
|
| 330 |
+
return len(_normalize_spaces(tag.get_text()))
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
# ── Readability engine ───────────────────────────────────────────────────────
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
class _Readability:
|
| 337 |
+
"""Internal engine that implements the readability extraction algorithm."""
|
| 338 |
+
|
| 339 |
+
# Tags to discard during article-extraction parsing. These are never
|
| 340 |
+
# part of the readable content and skipping them speeds up both tree
|
| 341 |
+
# construction and subsequent traversals.
|
| 342 |
+
_SKIP_TAGS = frozenset({"script", "style", "link", "noscript"})
|
| 343 |
+
|
| 344 |
+
def __init__(self, html: str, Soup: type, Tag: type) -> None:
|
| 345 |
+
self._raw_html = html
|
| 346 |
+
self._Soup = Soup
|
| 347 |
+
self._Tag = Tag
|
| 348 |
+
self._soup: Any = None
|
| 349 |
+
|
| 350 |
+
# ── Main entry point ─────────────────────────────────────────────────
|
| 351 |
+
|
| 352 |
+
def parse(self) -> ReadabilityResult:
|
| 353 |
+
"""Run the full extraction pipeline and return a result."""
|
| 354 |
+
# Parse once: extract metadata from the fresh DOM before mutations.
|
| 355 |
+
self._soup = self._Soup(self._raw_html)
|
| 356 |
+
metadata = self._extract_metadata(self._soup)
|
| 357 |
+
lang = self._detect_lang(self._soup)
|
| 358 |
+
direction = self._detect_dir(self._soup)
|
| 359 |
+
|
| 360 |
+
# Grab article content. The first iteration reuses self._soup
|
| 361 |
+
# (removing non-content tags in-place); retries use a faster
|
| 362 |
+
# parse that skips those tags during tree construction.
|
| 363 |
+
article_html, article_text = self._grab_article()
|
| 364 |
+
|
| 365 |
+
# If metadata title is empty, try to derive from article headings.
|
| 366 |
+
title = metadata.get("title", "")
|
| 367 |
+
if not title:
|
| 368 |
+
title = self._get_title_from_headings(self._soup) or ""
|
| 369 |
+
|
| 370 |
+
text = _normalize_spaces(article_text)
|
| 371 |
+
return ReadabilityResult(
|
| 372 |
+
title=title,
|
| 373 |
+
content=article_html,
|
| 374 |
+
text=text,
|
| 375 |
+
author=metadata.get("author"),
|
| 376 |
+
excerpt=metadata.get("excerpt"),
|
| 377 |
+
site_name=metadata.get("site_name"),
|
| 378 |
+
published_time=metadata.get("published_time"),
|
| 379 |
+
lang=lang,
|
| 380 |
+
dir=direction,
|
| 381 |
+
length=len(text),
|
| 382 |
+
)
|
| 383 |
+
|
| 384 |
+
# ── Article grabbing (with retry) ────────────────────────────────────
|
| 385 |
+
|
| 386 |
+
_PRE_CLEAN_TAGS = ["script", "style", "link", "noscript"]
|
| 387 |
+
|
| 388 |
+
def _grab_article(self) -> tuple[str, str]:
|
| 389 |
+
"""Extract article content, retrying with relaxed rules if needed.
|
| 390 |
+
|
| 391 |
+
Returns:
|
| 392 |
+
``(article_html, article_text)`` tuple.
|
| 393 |
+
"""
|
| 394 |
+
ruthless = True
|
| 395 |
+
|
| 396 |
+
for _attempt in range(2):
|
| 397 |
+
if _attempt == 0:
|
| 398 |
+
# First attempt: reuse the DOM already parsed by parse()
|
| 399 |
+
# and strip non-content tags in-place.
|
| 400 |
+
self._pre_clean()
|
| 401 |
+
else:
|
| 402 |
+
# Retry: fast re-parse that skips non-content tags at the
|
| 403 |
+
# parser level (avoids building + decomposing subtrees).
|
| 404 |
+
self._soup = self._Soup(self._raw_html, skip_tags=self._SKIP_TAGS)
|
| 405 |
+
|
| 406 |
+
if ruthless:
|
| 407 |
+
self._remove_unlikely_candidates()
|
| 408 |
+
|
| 409 |
+
self._transform_divs_to_paragraphs()
|
| 410 |
+
candidates = self._score_paragraphs()
|
| 411 |
+
|
| 412 |
+
best = self._select_best_candidate(candidates)
|
| 413 |
+
if best is not None:
|
| 414 |
+
article_tag = self._get_article(best, candidates)
|
| 415 |
+
self._sanitize(article_tag)
|
| 416 |
+
article_html = article_tag.to_html()
|
| 417 |
+
article_text = article_tag.get_text(separator=" ", strip=True)
|
| 418 |
+
|
| 419 |
+
if len(article_text) >= RETRY_LENGTH or not ruthless:
|
| 420 |
+
return article_html, article_text
|
| 421 |
+
|
| 422 |
+
# Too short — retry without ruthless filtering.
|
| 423 |
+
log.debug(
|
| 424 |
+
"Article too short (%d chars), retrying without "
|
| 425 |
+
"ruthless candidate removal",
|
| 426 |
+
len(article_text),
|
| 427 |
+
)
|
| 428 |
+
ruthless = False
|
| 429 |
+
continue
|
| 430 |
+
else:
|
| 431 |
+
if ruthless:
|
| 432 |
+
log.debug("No candidate found, retrying without ruthless")
|
| 433 |
+
ruthless = False
|
| 434 |
+
continue
|
| 435 |
+
# Fall back to body content.
|
| 436 |
+
break
|
| 437 |
+
|
| 438 |
+
# Final fallback: return body content as-is.
|
| 439 |
+
body = self._soup.find("body")
|
| 440 |
+
if body is not None:
|
| 441 |
+
return body.to_html(), body.get_text(separator=" ", strip=True)
|
| 442 |
+
return "", ""
|
| 443 |
+
|
| 444 |
+
# ── Pre-cleaning ─────────────────────────────────────────────────────
|
| 445 |
+
|
| 446 |
+
def _pre_clean(self) -> None:
|
| 447 |
+
"""Remove script, style, link and other non-content tags."""
|
| 448 |
+
for tag in list(self._soup.find_all(self._PRE_CLEAN_TAGS)):
|
| 449 |
+
tag.decompose()
|
| 450 |
+
|
| 451 |
+
# ── Metadata extraction ──────────────────────────────────────────────
|
| 452 |
+
|
| 453 |
+
def _extract_metadata(self, soup: Any) -> dict[str, str | None]:
|
| 454 |
+
"""Extract article metadata from JSON-LD and ``<meta>`` tags.
|
| 455 |
+
|
| 456 |
+
Args:
|
| 457 |
+
soup: A parsed ``Soup`` instance (unmutated).
|
| 458 |
+
|
| 459 |
+
Returns:
|
| 460 |
+
Dict with keys: title, author, excerpt, site_name, published_time.
|
| 461 |
+
"""
|
| 462 |
+
meta: dict[str, str | None] = {
|
| 463 |
+
"title": None,
|
| 464 |
+
"author": None,
|
| 465 |
+
"excerpt": None,
|
| 466 |
+
"site_name": None,
|
| 467 |
+
"published_time": None,
|
| 468 |
+
}
|
| 469 |
+
|
| 470 |
+
# 1. Try JSON-LD first (highest priority).
|
| 471 |
+
self._parse_jsonld(soup, meta)
|
| 472 |
+
|
| 473 |
+
# 2. Parse <meta> tags.
|
| 474 |
+
self._parse_meta_tags(soup, meta)
|
| 475 |
+
|
| 476 |
+
# 3. Title from <title> element (if not yet found).
|
| 477 |
+
if not meta["title"]:
|
| 478 |
+
title_tag = soup.find("title")
|
| 479 |
+
if title_tag:
|
| 480 |
+
meta["title"] = _normalize_spaces(title_tag.get_text())
|
| 481 |
+
|
| 482 |
+
# 4. Refine title by removing site name suffixes.
|
| 483 |
+
if meta["title"]:
|
| 484 |
+
meta["title"] = self._shorten_title(meta["title"])
|
| 485 |
+
|
| 486 |
+
return meta
|
| 487 |
+
|
| 488 |
+
def _parse_jsonld(self, soup: Any, meta: dict[str, str | None]) -> None:
|
| 489 |
+
"""Extract metadata from ``<script type="application/ld+json">``."""
|
| 490 |
+
for script in soup.find_all("script", attrs={"type": "application/ld+json"}):
|
| 491 |
+
text = script.get_text()
|
| 492 |
+
if not text:
|
| 493 |
+
continue
|
| 494 |
+
try:
|
| 495 |
+
data = json.loads(text)
|
| 496 |
+
except (json.JSONDecodeError, ValueError):
|
| 497 |
+
continue
|
| 498 |
+
|
| 499 |
+
# Handle @graph arrays.
|
| 500 |
+
if isinstance(data, dict) and "@graph" in data:
|
| 501 |
+
graph = data["@graph"]
|
| 502 |
+
if isinstance(graph, list):
|
| 503 |
+
for item in graph:
|
| 504 |
+
if self._is_article_jsonld(item):
|
| 505 |
+
data = item
|
| 506 |
+
break
|
| 507 |
+
else:
|
| 508 |
+
continue
|
| 509 |
+
|
| 510 |
+
if not self._is_article_jsonld(data):
|
| 511 |
+
continue
|
| 512 |
+
|
| 513 |
+
if not meta["title"]:
|
| 514 |
+
meta["title"] = _str_or_none(data.get("headline")) or _str_or_none(
|
| 515 |
+
data.get("name")
|
| 516 |
+
)
|
| 517 |
+
if not meta["author"]:
|
| 518 |
+
meta["author"] = self._extract_jsonld_author(data)
|
| 519 |
+
if not meta["excerpt"]:
|
| 520 |
+
meta["excerpt"] = _str_or_none(data.get("description"))
|
| 521 |
+
if not meta["published_time"]:
|
| 522 |
+
meta["published_time"] = _str_or_none(data.get("datePublished"))
|
| 523 |
+
if not meta["site_name"]:
|
| 524 |
+
publisher = data.get("publisher")
|
| 525 |
+
if isinstance(publisher, dict):
|
| 526 |
+
meta["site_name"] = _str_or_none(publisher.get("name"))
|
| 527 |
+
|
| 528 |
+
@staticmethod
|
| 529 |
+
def _is_article_jsonld(data: Any) -> bool:
|
| 530 |
+
"""Check if *data* is a Schema.org Article (or subtype)."""
|
| 531 |
+
if not isinstance(data, dict):
|
| 532 |
+
return False
|
| 533 |
+
schema_type = data.get("@type", "")
|
| 534 |
+
if isinstance(schema_type, list):
|
| 535 |
+
return any(t in JSONLD_ARTICLE_TYPES for t in schema_type)
|
| 536 |
+
return schema_type in JSONLD_ARTICLE_TYPES
|
| 537 |
+
|
| 538 |
+
@staticmethod
|
| 539 |
+
def _extract_jsonld_author(data: dict) -> str | None:
|
| 540 |
+
"""Extract author name from JSON-LD, handling various formats."""
|
| 541 |
+
author = data.get("author")
|
| 542 |
+
if author is None:
|
| 543 |
+
return None
|
| 544 |
+
if isinstance(author, str):
|
| 545 |
+
return author
|
| 546 |
+
if isinstance(author, dict):
|
| 547 |
+
return _str_or_none(author.get("name"))
|
| 548 |
+
if isinstance(author, list):
|
| 549 |
+
names = []
|
| 550 |
+
for a in author:
|
| 551 |
+
if isinstance(a, str):
|
| 552 |
+
names.append(a)
|
| 553 |
+
elif isinstance(a, dict) and "name" in a:
|
| 554 |
+
names.append(a["name"])
|
| 555 |
+
return ", ".join(names) if names else None
|
| 556 |
+
return None
|
| 557 |
+
|
| 558 |
+
def _parse_meta_tags(self, soup: Any, meta: dict[str, str | None]) -> None:
|
| 559 |
+
"""Extract metadata from ``<meta>`` tags (OpenGraph, DC, etc.)."""
|
| 560 |
+
# Mapping of meta property/name → target field + priority (lower wins).
|
| 561 |
+
property_map: dict[str, str] = {
|
| 562 |
+
"og:title": "title",
|
| 563 |
+
"og:description": "excerpt",
|
| 564 |
+
"og:site_name": "site_name",
|
| 565 |
+
"article:author": "author",
|
| 566 |
+
"article:published_time": "published_time",
|
| 567 |
+
"dc:title": "title",
|
| 568 |
+
"dc:creator": "author",
|
| 569 |
+
"dc:description": "excerpt",
|
| 570 |
+
"dcterm:title": "title",
|
| 571 |
+
"dcterm:creator": "author",
|
| 572 |
+
"dcterm:description": "excerpt",
|
| 573 |
+
"twitter:title": "title",
|
| 574 |
+
"twitter:description": "excerpt",
|
| 575 |
+
}
|
| 576 |
+
name_map: dict[str, str] = {
|
| 577 |
+
"author": "author",
|
| 578 |
+
"description": "excerpt",
|
| 579 |
+
"parsely-author": "author",
|
| 580 |
+
"parsely-pub-date": "published_time",
|
| 581 |
+
"parsely-title": "title",
|
| 582 |
+
}
|
| 583 |
+
|
| 584 |
+
for tag in soup.find_all("meta"):
|
| 585 |
+
content = tag.get("content", "")
|
| 586 |
+
if not content:
|
| 587 |
+
continue
|
| 588 |
+
content = _normalize_spaces(unescape(content))
|
| 589 |
+
|
| 590 |
+
prop = tag.get("property", "")
|
| 591 |
+
name = tag.get("name", "")
|
| 592 |
+
|
| 593 |
+
# Match by property attribute.
|
| 594 |
+
field = property_map.get(prop) or property_map.get(prop.lower())
|
| 595 |
+
if field and not meta[field]:
|
| 596 |
+
meta[field] = content
|
| 597 |
+
continue
|
| 598 |
+
|
| 599 |
+
# Match by name attribute.
|
| 600 |
+
field = name_map.get(name) or name_map.get(name.lower())
|
| 601 |
+
if field and not meta[field]:
|
| 602 |
+
meta[field] = content
|
| 603 |
+
|
| 604 |
+
@staticmethod
|
| 605 |
+
def _shorten_title(title: str) -> str:
|
| 606 |
+
"""Remove site name suffixes/prefixes from *title*.
|
| 607 |
+
|
| 608 |
+
Handles patterns like ``"Article Title | Site Name"`` by
|
| 609 |
+
splitting on common separators and picking the most likely
|
| 610 |
+
article title part.
|
| 611 |
+
"""
|
| 612 |
+
parts = TITLE_SEPARATORS_RE.split(title)
|
| 613 |
+
if len(parts) > 1:
|
| 614 |
+
# Heuristic: the longest part is usually the title, not
|
| 615 |
+
# the site name. But if the longest part is very short
|
| 616 |
+
# (< 2 words) and it's not significantly longer than the
|
| 617 |
+
# second-longest, keep the original.
|
| 618 |
+
sorted_parts = sorted(parts, key=len, reverse=True)
|
| 619 |
+
best = sorted_parts[0]
|
| 620 |
+
second = sorted_parts[1] if len(sorted_parts) > 1 else ""
|
| 621 |
+
# Use the longest part if it's meaningfully longer than
|
| 622 |
+
# the second part, OR if the first/last part is clearly
|
| 623 |
+
# the title (common patterns).
|
| 624 |
+
if len(best) > len(second):
|
| 625 |
+
return best.strip()
|
| 626 |
+
# If parts are roughly equal length, use the first one
|
| 627 |
+
# (title typically comes first).
|
| 628 |
+
return parts[0].strip()
|
| 629 |
+
|
| 630 |
+
# Try colon separator.
|
| 631 |
+
if ": " in title:
|
| 632 |
+
colon_parts = title.split(": ")
|
| 633 |
+
# Use text after colon if before-colon is short.
|
| 634 |
+
if len(colon_parts[0].split()) <= 5:
|
| 635 |
+
return ": ".join(colon_parts[1:]).strip()
|
| 636 |
+
|
| 637 |
+
return title.strip()
|
| 638 |
+
|
| 639 |
+
# ── Language / direction detection ────────────────────────────────────
|
| 640 |
+
|
| 641 |
+
@staticmethod
|
| 642 |
+
def _detect_lang(soup: Any) -> str | None:
|
| 643 |
+
"""Detect language from ``<html lang="...">``."""
|
| 644 |
+
html_tag = soup.find("html")
|
| 645 |
+
if html_tag is not None:
|
| 646 |
+
lang = html_tag.get("lang")
|
| 647 |
+
if lang:
|
| 648 |
+
return lang if isinstance(lang, str) else str(lang)
|
| 649 |
+
return None
|
| 650 |
+
|
| 651 |
+
@staticmethod
|
| 652 |
+
def _detect_dir(soup: Any) -> str | None:
|
| 653 |
+
"""Detect text direction from ``dir`` attribute."""
|
| 654 |
+
for tag_name in ("html", "body"):
|
| 655 |
+
tag = soup.find(tag_name)
|
| 656 |
+
if tag is not None:
|
| 657 |
+
d = tag.get("dir")
|
| 658 |
+
if d and isinstance(d, str) and d.lower() in ("ltr", "rtl"):
|
| 659 |
+
return d.lower()
|
| 660 |
+
return None
|
| 661 |
+
|
| 662 |
+
# ── Unlikely candidate removal ───────────────────────────────────────
|
| 663 |
+
|
| 664 |
+
def _remove_unlikely_candidates(self) -> None:
|
| 665 |
+
"""Remove elements whose class/id suggests non-content."""
|
| 666 |
+
for tag in list(self._soup._all_descendants()):
|
| 667 |
+
if tag.name in ("html", "body"):
|
| 668 |
+
continue
|
| 669 |
+
class_id = _get_class_id_string(tag)
|
| 670 |
+
if len(class_id) <= 1:
|
| 671 |
+
continue
|
| 672 |
+
if UNLIKELY_CANDIDATES_RE.search(
|
| 673 |
+
class_id
|
| 674 |
+
) and not OK_MAYBE_CANDIDATE_RE.search(class_id):
|
| 675 |
+
tag.decompose()
|
| 676 |
+
|
| 677 |
+
# ── Div-to-P transformation ──────────────────────────────────────────
|
| 678 |
+
|
| 679 |
+
def _transform_divs_to_paragraphs(self) -> None:
|
| 680 |
+
"""Convert ``<div>`` elements without block children into ``<p>``."""
|
| 681 |
+
for div in list(self._soup.find_all("div")):
|
| 682 |
+
has_block = any(
|
| 683 |
+
isinstance(c, self._Tag) and c.name in BLOCK_LEVEL_TAGS
|
| 684 |
+
for c in div.children
|
| 685 |
+
)
|
| 686 |
+
if not has_block:
|
| 687 |
+
div.name = "p"
|
| 688 |
+
|
| 689 |
+
# ── Scoring ──────────────────────────────────────────────────────────
|
| 690 |
+
|
| 691 |
+
def _score_paragraphs(self) -> dict[int, dict[str, Any]]:
|
| 692 |
+
"""Score paragraph-like nodes and propagate to ancestors.
|
| 693 |
+
|
| 694 |
+
Returns:
|
| 695 |
+
Dict mapping ``id(tag)`` → ``{"tag": tag, "score": float}``.
|
| 696 |
+
"""
|
| 697 |
+
candidates: dict[int, dict[str, Any]] = {}
|
| 698 |
+
|
| 699 |
+
for tag in self._soup.find_all(list(TAGS_TO_SCORE)):
|
| 700 |
+
inner_text = _normalize_spaces(tag.get_text())
|
| 701 |
+
if len(inner_text) < MIN_PARAGRAPH_LENGTH:
|
| 702 |
+
continue
|
| 703 |
+
|
| 704 |
+
parent = tag.parent
|
| 705 |
+
grandparent = parent.parent if parent is not None else None
|
| 706 |
+
|
| 707 |
+
# Ensure parent is initialised.
|
| 708 |
+
if parent is not None and id(parent) not in candidates:
|
| 709 |
+
candidates[id(parent)] = {
|
| 710 |
+
"tag": parent,
|
| 711 |
+
"score": self._init_score(parent),
|
| 712 |
+
}
|
| 713 |
+
# Ensure grandparent is initialised.
|
| 714 |
+
if grandparent is not None and id(grandparent) not in candidates:
|
| 715 |
+
candidates[id(grandparent)] = {
|
| 716 |
+
"tag": grandparent,
|
| 717 |
+
"score": self._init_score(grandparent),
|
| 718 |
+
}
|
| 719 |
+
|
| 720 |
+
# Content score for this paragraph.
|
| 721 |
+
inner_len = len(inner_text)
|
| 722 |
+
content_score = 1.0
|
| 723 |
+
content_score += len(COMMAS_RE.findall(inner_text))
|
| 724 |
+
content_score += min(inner_len / 100.0, 3.0)
|
| 725 |
+
|
| 726 |
+
# Propagate to parent (full) and grandparent (half).
|
| 727 |
+
if parent is not None and id(parent) in candidates:
|
| 728 |
+
candidates[id(parent)]["score"] += content_score
|
| 729 |
+
if grandparent is not None and id(grandparent) in candidates:
|
| 730 |
+
candidates[id(grandparent)]["score"] += content_score / 2.0
|
| 731 |
+
|
| 732 |
+
# Scale scores by link density.
|
| 733 |
+
for entry in candidates.values():
|
| 734 |
+
ld = self._get_link_density(entry["tag"])
|
| 735 |
+
entry["score"] *= 1.0 - ld
|
| 736 |
+
|
| 737 |
+
return candidates
|
| 738 |
+
|
| 739 |
+
def _init_score(self, tag: Any) -> float:
|
| 740 |
+
"""Compute initial score for a node based on tag name and class/id."""
|
| 741 |
+
score = float(TAG_WEIGHTS.get(tag.name, 0))
|
| 742 |
+
score += self._get_class_weight(tag)
|
| 743 |
+
return score
|
| 744 |
+
|
| 745 |
+
@staticmethod
|
| 746 |
+
def _get_class_weight(tag: Any) -> float:
|
| 747 |
+
"""Return ±25 weight based on class and id attribute content."""
|
| 748 |
+
weight = 0.0
|
| 749 |
+
for attr in ("class", "id"):
|
| 750 |
+
value = tag.get(attr, "")
|
| 751 |
+
if isinstance(value, list):
|
| 752 |
+
value = " ".join(value)
|
| 753 |
+
if not value:
|
| 754 |
+
continue
|
| 755 |
+
if NEGATIVE_RE.search(value):
|
| 756 |
+
weight -= 25.0
|
| 757 |
+
if POSITIVE_RE.search(value):
|
| 758 |
+
weight += 25.0
|
| 759 |
+
return weight
|
| 760 |
+
|
| 761 |
+
@staticmethod
|
| 762 |
+
def _get_link_density(tag: Any) -> float:
|
| 763 |
+
"""Ratio of link text length to total text length."""
|
| 764 |
+
total_len = _text_length(tag)
|
| 765 |
+
if total_len == 0:
|
| 766 |
+
return 0.0
|
| 767 |
+
link_len = sum(_text_length(a) for a in tag.find_all("a"))
|
| 768 |
+
return link_len / total_len
|
| 769 |
+
|
| 770 |
+
# ── Candidate selection ──────────────────────────────────────────────
|
| 771 |
+
|
| 772 |
+
@staticmethod
|
| 773 |
+
def _select_best_candidate(
|
| 774 |
+
candidates: dict[int, dict[str, Any]],
|
| 775 |
+
) -> dict[str, Any] | None:
|
| 776 |
+
"""Return the candidate with the highest score, or ``None``."""
|
| 777 |
+
if not candidates:
|
| 778 |
+
return None
|
| 779 |
+
return max(candidates.values(), key=lambda c: c["score"])
|
| 780 |
+
|
| 781 |
+
# ── Article assembly ─────────────────────────────────────────────────
|
| 782 |
+
|
| 783 |
+
def _get_article(
|
| 784 |
+
self, best: dict[str, Any], candidates: dict[int, dict[str, Any]]
|
| 785 |
+
) -> Any:
|
| 786 |
+
"""Build the article container from the best candidate + siblings.
|
| 787 |
+
|
| 788 |
+
Args:
|
| 789 |
+
best: The highest-scoring candidate dict.
|
| 790 |
+
candidates: All scored candidates.
|
| 791 |
+
|
| 792 |
+
Returns:
|
| 793 |
+
A new ``Tag`` containing the article content.
|
| 794 |
+
"""
|
| 795 |
+
best_tag = best["tag"]
|
| 796 |
+
parent = best_tag.parent
|
| 797 |
+
|
| 798 |
+
# Create an article wrapper.
|
| 799 |
+
article = self._Tag("div")
|
| 800 |
+
sibling_threshold = max(10.0, best["score"] * 0.2)
|
| 801 |
+
|
| 802 |
+
# If there's no parent, use the candidate itself.
|
| 803 |
+
if parent is None:
|
| 804 |
+
article.append(best_tag.extract())
|
| 805 |
+
return article
|
| 806 |
+
|
| 807 |
+
for sibling in list(parent.children):
|
| 808 |
+
if isinstance(sibling, str):
|
| 809 |
+
if sibling.strip():
|
| 810 |
+
article.append(sibling)
|
| 811 |
+
continue
|
| 812 |
+
|
| 813 |
+
should_include = False
|
| 814 |
+
|
| 815 |
+
if sibling is best_tag:
|
| 816 |
+
should_include = True
|
| 817 |
+
elif id(sibling) in candidates:
|
| 818 |
+
if candidates[id(sibling)]["score"] >= sibling_threshold:
|
| 819 |
+
should_include = True
|
| 820 |
+
elif isinstance(sibling, self._Tag) and sibling.name == "p":
|
| 821 |
+
link_density = self._get_link_density(sibling)
|
| 822 |
+
text = _normalize_spaces(sibling.get_text())
|
| 823 |
+
text_len = len(text)
|
| 824 |
+
if text_len > 80 and link_density < 0.25:
|
| 825 |
+
should_include = True
|
| 826 |
+
elif (
|
| 827 |
+
text_len <= 80 and link_density == 0 and re.search(r"\.\s*$", text)
|
| 828 |
+
):
|
| 829 |
+
should_include = True
|
| 830 |
+
|
| 831 |
+
if should_include:
|
| 832 |
+
article.append(sibling.extract())
|
| 833 |
+
|
| 834 |
+
return article
|
| 835 |
+
|
| 836 |
+
# ── Sanitization ─────────────────────────────────────────────────────
|
| 837 |
+
|
| 838 |
+
_HEADING_TAGS_SET = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})
|
| 839 |
+
# Combined list of form-related + heading tags for single find_all pass.
|
| 840 |
+
_REMOVE_AND_HEADING_TAGS = list(REMOVE_TAGS | _HEADING_TAGS_SET)
|
| 841 |
+
|
| 842 |
+
def _sanitize(self, article: Any) -> None:
|
| 843 |
+
"""Clean the extracted article of low-quality elements."""
|
| 844 |
+
# 1+2. Remove form-related elements and low-quality headings in
|
| 845 |
+
# a single find_all pass instead of two separate traversals.
|
| 846 |
+
for tag in list(article.find_all(self._REMOVE_AND_HEADING_TAGS)):
|
| 847 |
+
if tag.name in REMOVE_TAGS:
|
| 848 |
+
tag.decompose()
|
| 849 |
+
else:
|
| 850 |
+
# Heading tag — check quality.
|
| 851 |
+
if self._get_class_weight(tag) < 0:
|
| 852 |
+
tag.decompose()
|
| 853 |
+
elif self._get_link_density(tag) > 0.33:
|
| 854 |
+
tag.decompose()
|
| 855 |
+
|
| 856 |
+
# 3. Conditional cleanup of tables, divs, lists, etc. — single
|
| 857 |
+
# find_all with all candidate tags instead of per-tag-name loops.
|
| 858 |
+
self._clean_conditionally_all(article)
|
| 859 |
+
|
| 860 |
+
# 4. Remove empty tags.
|
| 861 |
+
self._remove_empty_tags(article)
|
| 862 |
+
|
| 863 |
+
_EMBED_TAGS = frozenset({"embed", "object", "iframe"})
|
| 864 |
+
|
| 865 |
+
@staticmethod
|
| 866 |
+
def _count_child_tags(tag: Any, embed_tags: frozenset[str]) -> tuple:
|
| 867 |
+
"""Count p, img, li, input, and embed descendant tags in one pass.
|
| 868 |
+
|
| 869 |
+
Args:
|
| 870 |
+
tag: The parent element to inspect.
|
| 871 |
+
embed_tags: Frozenset of tag names considered "embed-like".
|
| 872 |
+
|
| 873 |
+
Returns:
|
| 874 |
+
``(p_count, img_count, li_count, input_count, embed_count)``.
|
| 875 |
+
"""
|
| 876 |
+
p_count = img_count = li_count = input_count = embed_count = 0
|
| 877 |
+
for desc in tag._all_descendants():
|
| 878 |
+
dname = desc.name
|
| 879 |
+
if dname == "p":
|
| 880 |
+
p_count += 1
|
| 881 |
+
elif dname == "img":
|
| 882 |
+
img_count += 1
|
| 883 |
+
elif dname == "li":
|
| 884 |
+
li_count += 1
|
| 885 |
+
elif dname == "input":
|
| 886 |
+
input_count += 1
|
| 887 |
+
elif dname in embed_tags:
|
| 888 |
+
embed_count += 1
|
| 889 |
+
return p_count, img_count, li_count, input_count, embed_count
|
| 890 |
+
|
| 891 |
+
@staticmethod
|
| 892 |
+
def _should_remove_conditionally(
|
| 893 |
+
tag_name: str,
|
| 894 |
+
class_weight: float,
|
| 895 |
+
content_len: int,
|
| 896 |
+
link_density: float,
|
| 897 |
+
p_count: int,
|
| 898 |
+
img_count: int,
|
| 899 |
+
li_count: int,
|
| 900 |
+
input_count: int,
|
| 901 |
+
embed_count: int,
|
| 902 |
+
) -> bool:
|
| 903 |
+
"""Decide whether a conditionally-cleaned element should be removed.
|
| 904 |
+
|
| 905 |
+
Args:
|
| 906 |
+
tag_name: The tag name being evaluated.
|
| 907 |
+
class_weight: CSS class/id weight for the element.
|
| 908 |
+
content_len: Length of normalised inner text.
|
| 909 |
+
link_density: Ratio of link text to total text.
|
| 910 |
+
p_count: Number of ``<p>`` descendants.
|
| 911 |
+
img_count: Number of ``<img>`` descendants.
|
| 912 |
+
li_count: Number of ``<li>`` descendants.
|
| 913 |
+
input_count: Number of ``<input>`` descendants.
|
| 914 |
+
embed_count: Number of embed-like descendants.
|
| 915 |
+
|
| 916 |
+
Returns:
|
| 917 |
+
``True`` if the element should be removed.
|
| 918 |
+
"""
|
| 919 |
+
if img_count > 1 and p_count > 0 and (p_count / img_count) < 0.5:
|
| 920 |
+
return True
|
| 921 |
+
if li_count > p_count and tag_name not in ("ol", "ul"):
|
| 922 |
+
return True
|
| 923 |
+
if input_count > p_count / 3:
|
| 924 |
+
return True
|
| 925 |
+
if content_len < MIN_PARAGRAPH_LENGTH and (img_count == 0 or img_count > 2):
|
| 926 |
+
return True
|
| 927 |
+
if class_weight < 25 and link_density > 0.2:
|
| 928 |
+
return True
|
| 929 |
+
if class_weight >= 25 and link_density > 0.5:
|
| 930 |
+
return True
|
| 931 |
+
if (embed_count == 1 and content_len < 75) or (
|
| 932 |
+
embed_count > 1 and content_len < 200
|
| 933 |
+
):
|
| 934 |
+
return True
|
| 935 |
+
return False
|
| 936 |
+
|
| 937 |
+
_CLEAN_COND_TAGS_LIST = list(CLEAN_CONDITIONALLY_TAGS)
|
| 938 |
+
|
| 939 |
+
def _clean_conditionally_all(self, article: Any) -> None:
|
| 940 |
+
"""Remove conditionally-cleaned elements in a single find_all pass."""
|
| 941 |
+
for tag in list(article.find_all(self._CLEAN_COND_TAGS_LIST)):
|
| 942 |
+
class_weight = self._get_class_weight(tag)
|
| 943 |
+
# Quick reject: very negative weight.
|
| 944 |
+
if class_weight < -25:
|
| 945 |
+
tag.decompose()
|
| 946 |
+
continue
|
| 947 |
+
|
| 948 |
+
inner_text = _normalize_spaces(tag.get_text())
|
| 949 |
+
comma_count = len(COMMAS_RE.findall(inner_text))
|
| 950 |
+
|
| 951 |
+
# Commas indicate content-rich nodes; keep them.
|
| 952 |
+
if comma_count >= 10:
|
| 953 |
+
continue
|
| 954 |
+
|
| 955 |
+
counts = self._count_child_tags(tag, self._EMBED_TAGS)
|
| 956 |
+
content_len = len(inner_text)
|
| 957 |
+
link_density = self._get_link_density(tag)
|
| 958 |
+
|
| 959 |
+
if self._should_remove_conditionally(
|
| 960 |
+
tag.name, class_weight, content_len, link_density, *counts
|
| 961 |
+
):
|
| 962 |
+
tag.decompose()
|
| 963 |
+
|
| 964 |
+
@staticmethod
|
| 965 |
+
def _remove_empty_tags(article: Any) -> None:
|
| 966 |
+
"""Remove tags that have no text content and no images/embeds."""
|
| 967 |
+
keep_tags = frozenset(
|
| 968 |
+
{"img", "br", "hr", "embed", "object", "iframe", "video", "audio"}
|
| 969 |
+
)
|
| 970 |
+
# Process in reverse document order (children before parents) so
|
| 971 |
+
# that a single pass is sufficient.
|
| 972 |
+
for tag in reversed(article._all_descendants()):
|
| 973 |
+
if tag.name in keep_tags:
|
| 974 |
+
continue
|
| 975 |
+
if not tag.children:
|
| 976 |
+
tag.decompose()
|
| 977 |
+
elif not tag.get_text(strip=True) and not tag.find_all(list(keep_tags)):
|
| 978 |
+
tag.decompose()
|
| 979 |
+
|
| 980 |
+
# ── Title fallback from headings ─────────────────────────────────────
|
| 981 |
+
|
| 982 |
+
@staticmethod
|
| 983 |
+
def _get_title_from_headings(soup: Any) -> str | None:
|
| 984 |
+
"""Try to find a title from ``<h1>`` or ``<h2>`` headings."""
|
| 985 |
+
for level in ("h1", "h2"):
|
| 986 |
+
heading = soup.find(level)
|
| 987 |
+
if heading is not None:
|
| 988 |
+
text = _normalize_spaces(heading.get_text())
|
| 989 |
+
if text:
|
| 990 |
+
return text
|
| 991 |
+
return None
|
| 992 |
+
|
| 993 |
+
|
| 994 |
+
# ── Module-level helpers ─────────────────────────────────────────────────────
|
| 995 |
+
|
| 996 |
+
|
| 997 |
+
def _str_or_none(value: Any) -> str | None:
|
| 998 |
+
"""Return a non-empty stripped string or ``None``."""
|
| 999 |
+
if value is None:
|
| 1000 |
+
return None
|
| 1001 |
+
s = str(value).strip()
|
| 1002 |
+
return s if s else None
|
|
@@ -0,0 +1,503 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.3.0"
|
| 3 |
+
# deps = []
|
| 4 |
+
# tier = "simple"
|
| 5 |
+
# category = "process"
|
| 6 |
+
# note = "Install/update via `zerodep add retry`"
|
| 7 |
+
# ///
|
| 8 |
+
|
| 9 |
+
"""Zero-dependency retry with configurable backoff strategies.
|
| 10 |
+
|
| 11 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 12 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 13 |
+
|
| 14 |
+
Decorator-based retry with exponential / linear / fixed backoff,
|
| 15 |
+
jitter, exception and result filtering, and async support.
|
| 16 |
+
|
| 17 |
+
Basic usage::
|
| 18 |
+
|
| 19 |
+
@retry(max_retries=3)
|
| 20 |
+
def call_api():
|
| 21 |
+
return get("https://api.example.com/data")
|
| 22 |
+
|
| 23 |
+
Async usage::
|
| 24 |
+
|
| 25 |
+
@retry(max_retries=3, retry_on=(ConnectionError, TimeoutError))
|
| 26 |
+
async def call_api():
|
| 27 |
+
return await async_get("https://api.example.com/data")
|
| 28 |
+
|
| 29 |
+
Imperative usage::
|
| 30 |
+
|
| 31 |
+
result = retry_call(call_api, max_retries=5)
|
| 32 |
+
|
| 33 |
+
HTTP status filtering::
|
| 34 |
+
|
| 35 |
+
@retry(retry_on=retry_if_status(429, 502, 503))
|
| 36 |
+
def call_api():
|
| 37 |
+
resp = get("https://api.example.com/data")
|
| 38 |
+
resp.raise_for_status()
|
| 39 |
+
return resp
|
| 40 |
+
"""
|
| 41 |
+
|
| 42 |
+
from __future__ import annotations
|
| 43 |
+
|
| 44 |
+
import asyncio
|
| 45 |
+
import dataclasses
|
| 46 |
+
import functools
|
| 47 |
+
import inspect
|
| 48 |
+
import random
|
| 49 |
+
import time
|
| 50 |
+
from typing import Any, Callable
|
| 51 |
+
|
| 52 |
+
__all__ = [
|
| 53 |
+
# Constants
|
| 54 |
+
"DEFAULT_MAX_RETRIES",
|
| 55 |
+
"DEFAULT_BASE_DELAY",
|
| 56 |
+
"DEFAULT_MAX_DELAY",
|
| 57 |
+
"DEFAULT_BACKOFF_FACTOR",
|
| 58 |
+
# Exceptions
|
| 59 |
+
"RetryError",
|
| 60 |
+
# Data classes
|
| 61 |
+
"RetryState",
|
| 62 |
+
# Predicates
|
| 63 |
+
"retry_if_exception",
|
| 64 |
+
"retry_if_result",
|
| 65 |
+
"retry_if_status",
|
| 66 |
+
# Main API
|
| 67 |
+
"retry",
|
| 68 |
+
"retry_call",
|
| 69 |
+
]
|
| 70 |
+
|
| 71 |
+
# ── Defaults ──
|
| 72 |
+
|
| 73 |
+
DEFAULT_MAX_RETRIES = 3
|
| 74 |
+
DEFAULT_BASE_DELAY = 1.0
|
| 75 |
+
DEFAULT_MAX_DELAY = 60.0
|
| 76 |
+
DEFAULT_BACKOFF_FACTOR = 2.0
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
# ── Exceptions ──
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class RetryError(Exception):
|
| 83 |
+
"""Raised when all retry attempts are exhausted.
|
| 84 |
+
|
| 85 |
+
Attributes:
|
| 86 |
+
last_exception: The exception from the final attempt, or ``None``
|
| 87 |
+
if retries were triggered by result predicate.
|
| 88 |
+
attempts: Total number of calls made (initial + retries).
|
| 89 |
+
"""
|
| 90 |
+
|
| 91 |
+
def __init__(self, last_exception: BaseException | None, attempts: int) -> None:
|
| 92 |
+
self.last_exception = last_exception
|
| 93 |
+
self.attempts = attempts
|
| 94 |
+
super().__init__(
|
| 95 |
+
f"Retry exhausted after {attempts} attempt(s)"
|
| 96 |
+
+ (f": {last_exception}" if last_exception else "")
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
# ── Retry State ──
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
@dataclasses.dataclass
|
| 104 |
+
class RetryState:
|
| 105 |
+
"""Information about the current retry, passed to *on_retry* callback.
|
| 106 |
+
|
| 107 |
+
Attributes:
|
| 108 |
+
attempt: 1-based retry number (1 = first retry, not the initial call).
|
| 109 |
+
exception: The exception that triggered this retry, or ``None``.
|
| 110 |
+
result: The return value that triggered this retry, or ``None``.
|
| 111 |
+
delay: Seconds to sleep before the next attempt.
|
| 112 |
+
elapsed: Seconds elapsed since the initial call.
|
| 113 |
+
"""
|
| 114 |
+
|
| 115 |
+
attempt: int
|
| 116 |
+
exception: BaseException | None
|
| 117 |
+
result: Any
|
| 118 |
+
delay: float
|
| 119 |
+
elapsed: float
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
# ── Retry Condition Helpers ──
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def retry_if_exception(
|
| 126 |
+
*exc_types: type[BaseException],
|
| 127 |
+
) -> Callable[[BaseException], bool]:
|
| 128 |
+
"""Build a predicate that matches specific exception types.
|
| 129 |
+
|
| 130 |
+
Args:
|
| 131 |
+
*exc_types: Exception classes to retry on.
|
| 132 |
+
|
| 133 |
+
Returns:
|
| 134 |
+
A callable ``(exc) -> bool``.
|
| 135 |
+
"""
|
| 136 |
+
|
| 137 |
+
def predicate(exc: BaseException) -> bool:
|
| 138 |
+
return isinstance(exc, exc_types)
|
| 139 |
+
|
| 140 |
+
return predicate
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def retry_if_result(predicate: Callable[[Any], bool]) -> Callable[[Any], bool]:
|
| 144 |
+
"""Mark a callable as a result-retry predicate (identity helper).
|
| 145 |
+
|
| 146 |
+
Args:
|
| 147 |
+
predicate: A callable ``(result) -> bool`` returning ``True`` to retry.
|
| 148 |
+
|
| 149 |
+
Returns:
|
| 150 |
+
The same callable, for self-documenting call sites.
|
| 151 |
+
"""
|
| 152 |
+
return predicate
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def retry_if_status(*status_codes: int) -> Callable[[BaseException], bool]:
|
| 156 |
+
"""Build a predicate that retries on HTTP status codes.
|
| 157 |
+
|
| 158 |
+
Works with any exception carrying a ``status_code`` attribute
|
| 159 |
+
(e.g. ``httpclient.HTTPError``).
|
| 160 |
+
|
| 161 |
+
Args:
|
| 162 |
+
*status_codes: HTTP status codes to retry on (e.g. 429, 502, 503).
|
| 163 |
+
|
| 164 |
+
Returns:
|
| 165 |
+
A callable ``(exc) -> bool``.
|
| 166 |
+
"""
|
| 167 |
+
codes = set(status_codes)
|
| 168 |
+
|
| 169 |
+
def predicate(exc: BaseException) -> bool:
|
| 170 |
+
sc = getattr(exc, "status_code", None)
|
| 171 |
+
return sc is not None and sc in codes
|
| 172 |
+
|
| 173 |
+
return predicate
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
# ── Internal Helpers ──
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def _compute_delay(
|
| 180 |
+
attempt: int,
|
| 181 |
+
backoff: str,
|
| 182 |
+
base_delay: float,
|
| 183 |
+
backoff_factor: float,
|
| 184 |
+
max_delay: float,
|
| 185 |
+
jitter: str,
|
| 186 |
+
) -> float:
|
| 187 |
+
"""Compute the sleep duration for a given retry attempt.
|
| 188 |
+
|
| 189 |
+
Args:
|
| 190 |
+
attempt: 0-based retry index (0 = first retry).
|
| 191 |
+
backoff: Strategy name.
|
| 192 |
+
base_delay: Base delay in seconds.
|
| 193 |
+
backoff_factor: Multiplier for exponential backoff.
|
| 194 |
+
max_delay: Upper bound on delay.
|
| 195 |
+
jitter: Jitter mode.
|
| 196 |
+
|
| 197 |
+
Returns:
|
| 198 |
+
Sleep duration in seconds.
|
| 199 |
+
|
| 200 |
+
Raises:
|
| 201 |
+
ValueError: On unknown *backoff* or *jitter* value.
|
| 202 |
+
"""
|
| 203 |
+
if backoff == "exponential":
|
| 204 |
+
delay = base_delay * (backoff_factor**attempt)
|
| 205 |
+
elif backoff == "linear":
|
| 206 |
+
delay = base_delay * (attempt + 1)
|
| 207 |
+
elif backoff == "fixed":
|
| 208 |
+
delay = base_delay
|
| 209 |
+
else:
|
| 210 |
+
raise ValueError(f"Unknown backoff strategy: {backoff!r}")
|
| 211 |
+
|
| 212 |
+
delay = min(delay, max_delay)
|
| 213 |
+
|
| 214 |
+
if jitter == "full":
|
| 215 |
+
delay = random.uniform(0, delay)
|
| 216 |
+
elif jitter == "equal":
|
| 217 |
+
half = delay / 2
|
| 218 |
+
delay = half + random.uniform(0, half)
|
| 219 |
+
elif jitter == "none":
|
| 220 |
+
pass
|
| 221 |
+
else:
|
| 222 |
+
raise ValueError(f"Unknown jitter mode: {jitter!r}")
|
| 223 |
+
|
| 224 |
+
return delay
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def _should_retry_exception(
|
| 228 |
+
exc: BaseException,
|
| 229 |
+
retry_on: tuple[type[BaseException], ...] | Callable[[BaseException], bool],
|
| 230 |
+
) -> bool:
|
| 231 |
+
"""Check whether *exc* matches the retry condition."""
|
| 232 |
+
if isinstance(retry_on, tuple):
|
| 233 |
+
return isinstance(exc, retry_on)
|
| 234 |
+
return retry_on(exc)
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
# ── Core Retry Loops ──
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def _retry_sync(
|
| 241 |
+
fn: Callable[..., Any],
|
| 242 |
+
args: tuple[Any, ...],
|
| 243 |
+
kwargs: dict[str, Any],
|
| 244 |
+
*,
|
| 245 |
+
max_retries: int,
|
| 246 |
+
base_delay: float,
|
| 247 |
+
max_delay: float,
|
| 248 |
+
backoff: str,
|
| 249 |
+
backoff_factor: float,
|
| 250 |
+
jitter: str,
|
| 251 |
+
retry_on: tuple[type[BaseException], ...] | Callable[[BaseException], bool],
|
| 252 |
+
retry_on_result: Callable[[Any], bool] | None,
|
| 253 |
+
on_retry: Callable[[RetryState], None] | None,
|
| 254 |
+
) -> Any:
|
| 255 |
+
start = time.monotonic()
|
| 256 |
+
|
| 257 |
+
for attempt in range(max_retries + 1): # 0 = initial call
|
| 258 |
+
try:
|
| 259 |
+
result = fn(*args, **kwargs)
|
| 260 |
+
|
| 261 |
+
if retry_on_result is not None and retry_on_result(result):
|
| 262 |
+
if attempt >= max_retries:
|
| 263 |
+
raise RetryError(None, max_retries + 1)
|
| 264 |
+
delay = _compute_delay(
|
| 265 |
+
attempt, backoff, base_delay, backoff_factor, max_delay, jitter
|
| 266 |
+
)
|
| 267 |
+
if on_retry:
|
| 268 |
+
on_retry(
|
| 269 |
+
RetryState(
|
| 270 |
+
attempt=attempt + 1,
|
| 271 |
+
exception=None,
|
| 272 |
+
result=result,
|
| 273 |
+
delay=delay,
|
| 274 |
+
elapsed=time.monotonic() - start,
|
| 275 |
+
)
|
| 276 |
+
)
|
| 277 |
+
time.sleep(delay)
|
| 278 |
+
continue
|
| 279 |
+
|
| 280 |
+
return result
|
| 281 |
+
|
| 282 |
+
except BaseException as exc:
|
| 283 |
+
if not _should_retry_exception(exc, retry_on) or attempt >= max_retries:
|
| 284 |
+
raise
|
| 285 |
+
|
| 286 |
+
delay = _compute_delay(
|
| 287 |
+
attempt, backoff, base_delay, backoff_factor, max_delay, jitter
|
| 288 |
+
)
|
| 289 |
+
if on_retry:
|
| 290 |
+
on_retry(
|
| 291 |
+
RetryState(
|
| 292 |
+
attempt=attempt + 1,
|
| 293 |
+
exception=exc,
|
| 294 |
+
result=None,
|
| 295 |
+
delay=delay,
|
| 296 |
+
elapsed=time.monotonic() - start,
|
| 297 |
+
)
|
| 298 |
+
)
|
| 299 |
+
time.sleep(delay)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
async def _retry_async(
|
| 303 |
+
fn: Callable[..., Any],
|
| 304 |
+
args: tuple[Any, ...],
|
| 305 |
+
kwargs: dict[str, Any],
|
| 306 |
+
*,
|
| 307 |
+
max_retries: int,
|
| 308 |
+
base_delay: float,
|
| 309 |
+
max_delay: float,
|
| 310 |
+
backoff: str,
|
| 311 |
+
backoff_factor: float,
|
| 312 |
+
jitter: str,
|
| 313 |
+
retry_on: tuple[type[BaseException], ...] | Callable[[BaseException], bool],
|
| 314 |
+
retry_on_result: Callable[[Any], bool] | None,
|
| 315 |
+
on_retry: Callable[[RetryState], None] | None,
|
| 316 |
+
) -> Any:
|
| 317 |
+
start = time.monotonic()
|
| 318 |
+
|
| 319 |
+
for attempt in range(max_retries + 1):
|
| 320 |
+
try:
|
| 321 |
+
result = await fn(*args, **kwargs)
|
| 322 |
+
|
| 323 |
+
if retry_on_result is not None and retry_on_result(result):
|
| 324 |
+
if attempt >= max_retries:
|
| 325 |
+
raise RetryError(None, max_retries + 1)
|
| 326 |
+
delay = _compute_delay(
|
| 327 |
+
attempt, backoff, base_delay, backoff_factor, max_delay, jitter
|
| 328 |
+
)
|
| 329 |
+
if on_retry:
|
| 330 |
+
on_retry(
|
| 331 |
+
RetryState(
|
| 332 |
+
attempt=attempt + 1,
|
| 333 |
+
exception=None,
|
| 334 |
+
result=result,
|
| 335 |
+
delay=delay,
|
| 336 |
+
elapsed=time.monotonic() - start,
|
| 337 |
+
)
|
| 338 |
+
)
|
| 339 |
+
await asyncio.sleep(delay)
|
| 340 |
+
continue
|
| 341 |
+
|
| 342 |
+
return result
|
| 343 |
+
|
| 344 |
+
except BaseException as exc:
|
| 345 |
+
if not _should_retry_exception(exc, retry_on) or attempt >= max_retries:
|
| 346 |
+
raise
|
| 347 |
+
|
| 348 |
+
delay = _compute_delay(
|
| 349 |
+
attempt, backoff, base_delay, backoff_factor, max_delay, jitter
|
| 350 |
+
)
|
| 351 |
+
if on_retry:
|
| 352 |
+
on_retry(
|
| 353 |
+
RetryState(
|
| 354 |
+
attempt=attempt + 1,
|
| 355 |
+
exception=exc,
|
| 356 |
+
result=None,
|
| 357 |
+
delay=delay,
|
| 358 |
+
elapsed=time.monotonic() - start,
|
| 359 |
+
)
|
| 360 |
+
)
|
| 361 |
+
await asyncio.sleep(delay)
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
# ── Public API ──
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
def retry(
|
| 368 |
+
fn: Callable[..., Any] | None = None,
|
| 369 |
+
*,
|
| 370 |
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
| 371 |
+
base_delay: float = DEFAULT_BASE_DELAY,
|
| 372 |
+
max_delay: float = DEFAULT_MAX_DELAY,
|
| 373 |
+
backoff: str = "exponential",
|
| 374 |
+
backoff_factor: float = DEFAULT_BACKOFF_FACTOR,
|
| 375 |
+
jitter: str = "full",
|
| 376 |
+
retry_on: tuple[type[BaseException], ...] | Callable[[BaseException], bool] = (
|
| 377 |
+
Exception,
|
| 378 |
+
),
|
| 379 |
+
retry_on_result: Callable[[Any], bool] | None = None,
|
| 380 |
+
on_retry: Callable[[RetryState], None] | None = None,
|
| 381 |
+
) -> Callable[..., Any]:
|
| 382 |
+
"""Decorator that retries a function on failure with configurable backoff.
|
| 383 |
+
|
| 384 |
+
Can be used with or without arguments::
|
| 385 |
+
|
| 386 |
+
@retry
|
| 387 |
+
def f(): ...
|
| 388 |
+
|
| 389 |
+
@retry()
|
| 390 |
+
def g(): ...
|
| 391 |
+
|
| 392 |
+
@retry(max_retries=5, backoff="linear")
|
| 393 |
+
def h(): ...
|
| 394 |
+
|
| 395 |
+
Automatically detects async functions and uses ``asyncio.sleep``.
|
| 396 |
+
|
| 397 |
+
Args:
|
| 398 |
+
fn: The function to decorate (set automatically when used as
|
| 399 |
+
``@retry`` without parentheses).
|
| 400 |
+
max_retries: Maximum number of retries (not counting the initial call).
|
| 401 |
+
base_delay: Base delay in seconds before the first retry.
|
| 402 |
+
max_delay: Upper bound on computed delay.
|
| 403 |
+
backoff: Backoff strategy — ``"exponential"``, ``"linear"``, or
|
| 404 |
+
``"fixed"``.
|
| 405 |
+
backoff_factor: Multiplier for exponential backoff.
|
| 406 |
+
jitter: Jitter mode — ``"full"`` (uniform [0, delay]),
|
| 407 |
+
``"equal"`` (delay/2 + uniform [0, delay/2]), or ``"none"``.
|
| 408 |
+
retry_on: Exception types or a callable ``(exc) -> bool`` deciding
|
| 409 |
+
whether to retry. Defaults to ``(Exception,)``.
|
| 410 |
+
retry_on_result: Optional callable ``(result) -> bool``. When it
|
| 411 |
+
returns ``True`` the call is retried.
|
| 412 |
+
on_retry: Optional callback invoked before each retry sleep with a
|
| 413 |
+
:class:`RetryState` instance.
|
| 414 |
+
|
| 415 |
+
Returns:
|
| 416 |
+
The decorated function (sync or async, matching the original).
|
| 417 |
+
|
| 418 |
+
Raises:
|
| 419 |
+
RetryError: When retries are exhausted due to *retry_on_result*.
|
| 420 |
+
The original exception: When retries are exhausted due to exceptions.
|
| 421 |
+
"""
|
| 422 |
+
|
| 423 |
+
def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
|
| 424 |
+
if inspect.iscoroutinefunction(fn):
|
| 425 |
+
|
| 426 |
+
@functools.wraps(fn)
|
| 427 |
+
async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 428 |
+
return await _retry_async(
|
| 429 |
+
fn,
|
| 430 |
+
args,
|
| 431 |
+
kwargs,
|
| 432 |
+
max_retries=max_retries,
|
| 433 |
+
base_delay=base_delay,
|
| 434 |
+
max_delay=max_delay,
|
| 435 |
+
backoff=backoff,
|
| 436 |
+
backoff_factor=backoff_factor,
|
| 437 |
+
jitter=jitter,
|
| 438 |
+
retry_on=retry_on,
|
| 439 |
+
retry_on_result=retry_on_result,
|
| 440 |
+
on_retry=on_retry,
|
| 441 |
+
)
|
| 442 |
+
|
| 443 |
+
return async_wrapper
|
| 444 |
+
|
| 445 |
+
@functools.wraps(fn)
|
| 446 |
+
def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 447 |
+
return _retry_sync(
|
| 448 |
+
fn,
|
| 449 |
+
args,
|
| 450 |
+
kwargs,
|
| 451 |
+
max_retries=max_retries,
|
| 452 |
+
base_delay=base_delay,
|
| 453 |
+
max_delay=max_delay,
|
| 454 |
+
backoff=backoff,
|
| 455 |
+
backoff_factor=backoff_factor,
|
| 456 |
+
jitter=jitter,
|
| 457 |
+
retry_on=retry_on,
|
| 458 |
+
retry_on_result=retry_on_result,
|
| 459 |
+
on_retry=on_retry,
|
| 460 |
+
)
|
| 461 |
+
|
| 462 |
+
return sync_wrapper
|
| 463 |
+
|
| 464 |
+
if fn is not None:
|
| 465 |
+
return decorator(fn)
|
| 466 |
+
return decorator
|
| 467 |
+
|
| 468 |
+
|
| 469 |
+
def retry_call(
|
| 470 |
+
fn: Callable[..., Any],
|
| 471 |
+
args: tuple[Any, ...] = (),
|
| 472 |
+
kwargs: dict[str, Any] | None = None,
|
| 473 |
+
**retry_kwargs: Any,
|
| 474 |
+
) -> Any:
|
| 475 |
+
"""Call *fn* with retry logic without using a decorator.
|
| 476 |
+
|
| 477 |
+
Args:
|
| 478 |
+
fn: The callable to invoke.
|
| 479 |
+
args: Positional arguments for *fn*.
|
| 480 |
+
kwargs: Keyword arguments for *fn*.
|
| 481 |
+
**retry_kwargs: Same keyword arguments accepted by :func:`retry`.
|
| 482 |
+
|
| 483 |
+
Returns:
|
| 484 |
+
The return value of *fn* on success.
|
| 485 |
+
"""
|
| 486 |
+
if kwargs is None:
|
| 487 |
+
kwargs = {}
|
| 488 |
+
|
| 489 |
+
opts: dict[str, Any] = dict(
|
| 490 |
+
max_retries=retry_kwargs.pop("max_retries", DEFAULT_MAX_RETRIES),
|
| 491 |
+
base_delay=retry_kwargs.pop("base_delay", DEFAULT_BASE_DELAY),
|
| 492 |
+
max_delay=retry_kwargs.pop("max_delay", DEFAULT_MAX_DELAY),
|
| 493 |
+
backoff=retry_kwargs.pop("backoff", "exponential"),
|
| 494 |
+
backoff_factor=retry_kwargs.pop("backoff_factor", DEFAULT_BACKOFF_FACTOR),
|
| 495 |
+
jitter=retry_kwargs.pop("jitter", "full"),
|
| 496 |
+
retry_on=retry_kwargs.pop("retry_on", (Exception,)),
|
| 497 |
+
retry_on_result=retry_kwargs.pop("retry_on_result", None),
|
| 498 |
+
on_retry=retry_kwargs.pop("on_retry", None),
|
| 499 |
+
)
|
| 500 |
+
|
| 501 |
+
if inspect.iscoroutinefunction(fn):
|
| 502 |
+
return _retry_async(fn, args, kwargs, **opts)
|
| 503 |
+
return _retry_sync(fn, args, kwargs, **opts)
|
|
@@ -0,0 +1,998 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.6.0"
|
| 3 |
+
# deps = []
|
| 4 |
+
# tier = "medium"
|
| 5 |
+
# category = "text"
|
| 6 |
+
# note = "Install/update via `zerodep add soup`"
|
| 7 |
+
# ///
|
| 8 |
+
|
| 9 |
+
"""HTML parser with BeautifulSoup-like API — zero-dep, stdlib only, Python 3.10+.
|
| 10 |
+
|
| 11 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 12 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 13 |
+
|
| 14 |
+
Provides a lightweight DOM tree built on top of ``html.parser.HTMLParser``.
|
| 15 |
+
Supports ``find``, ``find_all``, ``select``, ``select_one``, ``get_text``,
|
| 16 |
+
``decompose``, and ``find_parent`` — the subset of BeautifulSoup used by
|
| 17 |
+
the vast majority of real-world scraping scripts.
|
| 18 |
+
|
| 19 |
+
Supports CSS pseudo-selectors: ``:first-child``, ``:last-child``,
|
| 20 |
+
``:only-child``, and ``:not(selector)``.
|
| 21 |
+
|
| 22 |
+
Does NOT implement: ``.prettify()``, ``.stripped_strings``, ``.descendants``
|
| 23 |
+
iterator, ``.next_sibling`` / ``.previous_sibling``, ``NavigableString`` class,
|
| 24 |
+
multiple parser backends.
|
| 25 |
+
|
| 26 |
+
Example::
|
| 27 |
+
|
| 28 |
+
soup = Soup("<html><body><p class='msg'>Hello</p></body></html>")
|
| 29 |
+
print(soup.find("p", class_="msg").text)
|
| 30 |
+
# Hello
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
from __future__ import annotations
|
| 34 |
+
|
| 35 |
+
import re
|
| 36 |
+
from html.parser import HTMLParser
|
| 37 |
+
from typing import Any
|
| 38 |
+
|
| 39 |
+
__all__ = [
|
| 40 |
+
"SELF_CLOSING_TAGS",
|
| 41 |
+
"Tag",
|
| 42 |
+
"Soup",
|
| 43 |
+
]
|
| 44 |
+
|
| 45 |
+
# ── Constants ─────────────────────────────────────────────────────────────────
|
| 46 |
+
|
| 47 |
+
SELF_CLOSING_TAGS = frozenset(
|
| 48 |
+
{
|
| 49 |
+
"area",
|
| 50 |
+
"base",
|
| 51 |
+
"br",
|
| 52 |
+
"col",
|
| 53 |
+
"embed",
|
| 54 |
+
"hr",
|
| 55 |
+
"img",
|
| 56 |
+
"input",
|
| 57 |
+
"link",
|
| 58 |
+
"meta",
|
| 59 |
+
"param",
|
| 60 |
+
"source",
|
| 61 |
+
"track",
|
| 62 |
+
"wbr",
|
| 63 |
+
}
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
# ── Tag ───────────────────────────────────────────────────────────────────────
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class Tag:
|
| 70 |
+
"""A single HTML element node.
|
| 71 |
+
|
| 72 |
+
Attributes:
|
| 73 |
+
name: Tag name (e.g. ``"div"``).
|
| 74 |
+
attrs: Dictionary of attribute name to value. The ``class`` attribute
|
| 75 |
+
is stored as a **list** of class names; all others as ``str``.
|
| 76 |
+
children: Ordered child nodes — either ``Tag`` or plain ``str``.
|
| 77 |
+
parent: Parent ``Tag``, or ``None`` for the root document.
|
| 78 |
+
"""
|
| 79 |
+
|
| 80 |
+
__slots__ = ("name", "attrs", "children", "parent")
|
| 81 |
+
|
| 82 |
+
def __init__(
|
| 83 |
+
self,
|
| 84 |
+
name: str,
|
| 85 |
+
attrs: dict[str, str | list[str]] | None = None,
|
| 86 |
+
parent: Tag | None = None,
|
| 87 |
+
) -> None:
|
| 88 |
+
self.name: str = name
|
| 89 |
+
self.attrs: dict[str, str | list[str]] = attrs if attrs is not None else {}
|
| 90 |
+
self.children: list[Tag | str] = []
|
| 91 |
+
self.parent: Tag | None = parent
|
| 92 |
+
|
| 93 |
+
# ── Attribute access ──────────────────────────────────────────────────
|
| 94 |
+
|
| 95 |
+
def get(self, attr: str, default: Any = None) -> Any:
|
| 96 |
+
"""Return attribute value, or *default* if not present."""
|
| 97 |
+
return self.attrs.get(attr, default)
|
| 98 |
+
|
| 99 |
+
def __getitem__(self, attr: str) -> Any:
|
| 100 |
+
"""Return attribute value; raise ``KeyError`` if missing."""
|
| 101 |
+
return self.attrs[attr]
|
| 102 |
+
|
| 103 |
+
def __contains__(self, attr: str) -> bool:
|
| 104 |
+
return attr in self.attrs
|
| 105 |
+
|
| 106 |
+
def __setitem__(self, attr: str, value: Any) -> None:
|
| 107 |
+
"""Set an attribute value (e.g. ``tag['id'] = 'main'``)."""
|
| 108 |
+
self.attrs[attr] = value
|
| 109 |
+
|
| 110 |
+
def __delitem__(self, attr: str) -> None:
|
| 111 |
+
"""Delete an attribute (e.g. ``del tag['id']``)."""
|
| 112 |
+
del self.attrs[attr]
|
| 113 |
+
|
| 114 |
+
# ── Text helpers ──────────────────────────────────────────────────────
|
| 115 |
+
|
| 116 |
+
@property
|
| 117 |
+
def text(self) -> str:
|
| 118 |
+
"""Concatenated text content of this element and all descendants."""
|
| 119 |
+
return self.get_text()
|
| 120 |
+
|
| 121 |
+
@property
|
| 122 |
+
def string(self) -> str | None:
|
| 123 |
+
"""If this element has exactly one text child (possibly nested), return it.
|
| 124 |
+
|
| 125 |
+
Returns ``None`` when the element has no children, multiple children,
|
| 126 |
+
or a mix of text and tags.
|
| 127 |
+
"""
|
| 128 |
+
# Direct single-text child
|
| 129 |
+
if len(self.children) == 1:
|
| 130 |
+
child = self.children[0]
|
| 131 |
+
if isinstance(child, str):
|
| 132 |
+
return child
|
| 133 |
+
return child.string
|
| 134 |
+
# No children or multiple children -> None
|
| 135 |
+
return None
|
| 136 |
+
|
| 137 |
+
def get_text(self, separator: str = "", strip: bool = False) -> str:
|
| 138 |
+
"""Return all text under this element concatenated.
|
| 139 |
+
|
| 140 |
+
Args:
|
| 141 |
+
separator: Inserted between text fragments.
|
| 142 |
+
strip: If ``True`` each fragment is whitespace-stripped and empty
|
| 143 |
+
fragments are dropped.
|
| 144 |
+
|
| 145 |
+
Returns:
|
| 146 |
+
The combined text.
|
| 147 |
+
"""
|
| 148 |
+
parts: list[str] = []
|
| 149 |
+
self._collect_text(parts)
|
| 150 |
+
if strip:
|
| 151 |
+
parts = [p.strip() for p in parts]
|
| 152 |
+
parts = [p for p in parts if p]
|
| 153 |
+
return separator.join(parts)
|
| 154 |
+
|
| 155 |
+
def _collect_text(self, acc: list[str]) -> None:
|
| 156 |
+
for child in self.children:
|
| 157 |
+
if isinstance(child, str):
|
| 158 |
+
acc.append(child)
|
| 159 |
+
else:
|
| 160 |
+
child._collect_text(acc)
|
| 161 |
+
|
| 162 |
+
# ── Tree modification ─────────────────────────────────────────────────
|
| 163 |
+
|
| 164 |
+
def append(self, child: Tag | str) -> None:
|
| 165 |
+
"""Append *child* to this element's children.
|
| 166 |
+
|
| 167 |
+
If *child* is a ``Tag`` already attached to a parent, it is first
|
| 168 |
+
detached from its old position.
|
| 169 |
+
|
| 170 |
+
Args:
|
| 171 |
+
child: A ``Tag`` or plain text string to append.
|
| 172 |
+
"""
|
| 173 |
+
if isinstance(child, Tag):
|
| 174 |
+
if child.parent is not None:
|
| 175 |
+
try:
|
| 176 |
+
child.parent.children.remove(child)
|
| 177 |
+
except ValueError:
|
| 178 |
+
pass
|
| 179 |
+
child.parent = self
|
| 180 |
+
self.children.append(child)
|
| 181 |
+
|
| 182 |
+
def insert(self, index: int, child: Tag | str) -> None:
|
| 183 |
+
"""Insert *child* at *index* in this element's children.
|
| 184 |
+
|
| 185 |
+
Args:
|
| 186 |
+
index: Position to insert at (same semantics as ``list.insert``).
|
| 187 |
+
child: A ``Tag`` or plain text string to insert.
|
| 188 |
+
"""
|
| 189 |
+
if isinstance(child, Tag):
|
| 190 |
+
if child.parent is not None:
|
| 191 |
+
try:
|
| 192 |
+
child.parent.children.remove(child)
|
| 193 |
+
except ValueError:
|
| 194 |
+
pass
|
| 195 |
+
child.parent = self
|
| 196 |
+
self.children.insert(index, child)
|
| 197 |
+
|
| 198 |
+
def extract(self) -> Tag:
|
| 199 |
+
"""Remove this element from its parent but keep its content intact.
|
| 200 |
+
|
| 201 |
+
Unlike ``decompose``, the element and its subtree remain usable
|
| 202 |
+
after extraction.
|
| 203 |
+
|
| 204 |
+
Returns:
|
| 205 |
+
This element (now detached).
|
| 206 |
+
"""
|
| 207 |
+
if self.parent is not None:
|
| 208 |
+
try:
|
| 209 |
+
self.parent.children.remove(self)
|
| 210 |
+
except ValueError:
|
| 211 |
+
pass
|
| 212 |
+
self.parent = None
|
| 213 |
+
return self
|
| 214 |
+
|
| 215 |
+
def replace_with(self, new_node: Tag | str) -> Tag:
|
| 216 |
+
"""Replace this element with *new_node* in the parent's children.
|
| 217 |
+
|
| 218 |
+
Args:
|
| 219 |
+
new_node: The replacement ``Tag`` or text string.
|
| 220 |
+
|
| 221 |
+
Returns:
|
| 222 |
+
This element (now detached).
|
| 223 |
+
|
| 224 |
+
Raises:
|
| 225 |
+
ValueError: If the element has no parent.
|
| 226 |
+
"""
|
| 227 |
+
if self.parent is None:
|
| 228 |
+
raise ValueError("Cannot replace a detached element")
|
| 229 |
+
parent = self.parent
|
| 230 |
+
for i, child in enumerate(parent.children):
|
| 231 |
+
if child is self:
|
| 232 |
+
parent.children[i] = new_node
|
| 233 |
+
if isinstance(new_node, Tag):
|
| 234 |
+
if new_node.parent is not None:
|
| 235 |
+
try:
|
| 236 |
+
new_node.parent.children.remove(new_node)
|
| 237 |
+
except ValueError:
|
| 238 |
+
pass
|
| 239 |
+
new_node.parent = parent
|
| 240 |
+
self.parent = None
|
| 241 |
+
return self
|
| 242 |
+
raise ValueError("Element not found in parent's children") # pragma: no cover
|
| 243 |
+
|
| 244 |
+
def unwrap(self) -> None:
|
| 245 |
+
"""Remove this tag but keep its children (re-parent them).
|
| 246 |
+
|
| 247 |
+
The children are spliced into the parent's children list at the
|
| 248 |
+
position formerly occupied by this element.
|
| 249 |
+
"""
|
| 250 |
+
if self.parent is None:
|
| 251 |
+
return
|
| 252 |
+
parent = self.parent
|
| 253 |
+
idx = next(i for i, c in enumerate(parent.children) if c is self)
|
| 254 |
+
# Splice children into parent at the position of this element.
|
| 255 |
+
for child in self.children:
|
| 256 |
+
if isinstance(child, Tag):
|
| 257 |
+
child.parent = parent
|
| 258 |
+
parent.children[idx : idx + 1] = self.children
|
| 259 |
+
self.children = []
|
| 260 |
+
self.parent = None
|
| 261 |
+
|
| 262 |
+
def decompose(self) -> None:
|
| 263 |
+
"""Remove this element from its parent and discard its content."""
|
| 264 |
+
if self.parent is not None:
|
| 265 |
+
try:
|
| 266 |
+
self.parent.children.remove(self)
|
| 267 |
+
except ValueError:
|
| 268 |
+
pass
|
| 269 |
+
self.parent = None
|
| 270 |
+
self.children.clear()
|
| 271 |
+
|
| 272 |
+
# ── Searching ─────────────────────────────────────────────────────────
|
| 273 |
+
|
| 274 |
+
def find(
|
| 275 |
+
self,
|
| 276 |
+
name: str | list[str] | None = None,
|
| 277 |
+
attrs: dict[str, str | bool] | None = None,
|
| 278 |
+
*,
|
| 279 |
+
class_: str | None = None,
|
| 280 |
+
**kwargs: str | bool,
|
| 281 |
+
) -> Tag | None:
|
| 282 |
+
"""Return the first descendant matching the criteria, or ``None``.
|
| 283 |
+
|
| 284 |
+
Args:
|
| 285 |
+
name: Tag name(s) to match. ``None`` matches any tag.
|
| 286 |
+
attrs: Dict of attribute filters.
|
| 287 |
+
class_: Shorthand for ``attrs={"class": value}``.
|
| 288 |
+
**kwargs: Extra attribute filters (``href=True`` means *has* href).
|
| 289 |
+
|
| 290 |
+
Returns:
|
| 291 |
+
The first matching ``Tag``, or ``None``.
|
| 292 |
+
"""
|
| 293 |
+
results = self.find_all(name, attrs, class_=class_, limit=1, **kwargs)
|
| 294 |
+
return results[0] if results else None
|
| 295 |
+
|
| 296 |
+
def find_all(
|
| 297 |
+
self,
|
| 298 |
+
name: str | list[str] | None = None,
|
| 299 |
+
attrs: dict[str, str | bool] | None = None,
|
| 300 |
+
*,
|
| 301 |
+
class_: str | None = None,
|
| 302 |
+
limit: int | None = None,
|
| 303 |
+
**kwargs: str | bool,
|
| 304 |
+
) -> list[Tag]:
|
| 305 |
+
"""Return all descendants matching the criteria.
|
| 306 |
+
|
| 307 |
+
Args:
|
| 308 |
+
name: Tag name(s) to match.
|
| 309 |
+
attrs: Dict of attribute filters.
|
| 310 |
+
class_: Shorthand for ``attrs={"class": value}``.
|
| 311 |
+
limit: Stop after finding this many results.
|
| 312 |
+
**kwargs: Extra attribute filters.
|
| 313 |
+
|
| 314 |
+
Returns:
|
| 315 |
+
A list of matching ``Tag`` objects.
|
| 316 |
+
"""
|
| 317 |
+
merged = dict(attrs) if attrs else {}
|
| 318 |
+
if class_ is not None:
|
| 319 |
+
merged["class"] = class_
|
| 320 |
+
merged.update(kwargs)
|
| 321 |
+
|
| 322 |
+
# Fast path: name-only search with no attribute filters.
|
| 323 |
+
if not merged:
|
| 324 |
+
if isinstance(name, str):
|
| 325 |
+
results: list[Tag] = []
|
| 326 |
+
self._search_by_single_name(name, results, limit)
|
| 327 |
+
return results
|
| 328 |
+
if isinstance(name, list):
|
| 329 |
+
name_set: frozenset[str] = frozenset(name)
|
| 330 |
+
results = []
|
| 331 |
+
self._search_by_name_set(name_set, results, limit)
|
| 332 |
+
return results
|
| 333 |
+
|
| 334 |
+
if isinstance(name, list):
|
| 335 |
+
name_set = frozenset(name)
|
| 336 |
+
else:
|
| 337 |
+
name_set = None # type: ignore[assignment]
|
| 338 |
+
|
| 339 |
+
results = []
|
| 340 |
+
self._search(name, name_set, merged, results, limit)
|
| 341 |
+
return results
|
| 342 |
+
|
| 343 |
+
def __call__(self, *args: Any, **kwargs: Any) -> list[Tag]:
|
| 344 |
+
"""Calling a tag is equivalent to ``find_all``."""
|
| 345 |
+
return self.find_all(*args, **kwargs)
|
| 346 |
+
|
| 347 |
+
def _search(
|
| 348 |
+
self,
|
| 349 |
+
name: str | list[str] | None,
|
| 350 |
+
name_set: frozenset[str] | None,
|
| 351 |
+
attr_filters: dict[str, str | bool],
|
| 352 |
+
results: list[Tag],
|
| 353 |
+
limit: int | None,
|
| 354 |
+
) -> None:
|
| 355 |
+
for child in self.children:
|
| 356 |
+
if limit is not None and len(results) >= limit:
|
| 357 |
+
return
|
| 358 |
+
if isinstance(child, Tag):
|
| 359 |
+
if _matches(child, name, name_set, attr_filters):
|
| 360 |
+
results.append(child)
|
| 361 |
+
if limit is not None and len(results) >= limit:
|
| 362 |
+
return
|
| 363 |
+
child._search(name, name_set, attr_filters, results, limit)
|
| 364 |
+
|
| 365 |
+
def _search_by_name_set(
|
| 366 |
+
self,
|
| 367 |
+
name_set: frozenset[str],
|
| 368 |
+
results: list[Tag],
|
| 369 |
+
limit: int | None,
|
| 370 |
+
) -> None:
|
| 371 |
+
"""Fast path for searching by a set of tag names with no attr filters."""
|
| 372 |
+
for child in self.children:
|
| 373 |
+
if limit is not None and len(results) >= limit:
|
| 374 |
+
return
|
| 375 |
+
if isinstance(child, Tag):
|
| 376 |
+
if child.name in name_set:
|
| 377 |
+
results.append(child)
|
| 378 |
+
if limit is not None and len(results) >= limit:
|
| 379 |
+
return
|
| 380 |
+
child._search_by_name_set(name_set, results, limit)
|
| 381 |
+
|
| 382 |
+
def _search_by_single_name(
|
| 383 |
+
self,
|
| 384 |
+
name: str,
|
| 385 |
+
results: list[Tag],
|
| 386 |
+
limit: int | None,
|
| 387 |
+
) -> None:
|
| 388 |
+
"""Fast path for searching by a single tag name with no attr filters."""
|
| 389 |
+
for child in self.children:
|
| 390 |
+
if limit is not None and len(results) >= limit:
|
| 391 |
+
return
|
| 392 |
+
if isinstance(child, Tag):
|
| 393 |
+
if child.name == name:
|
| 394 |
+
results.append(child)
|
| 395 |
+
if limit is not None and len(results) >= limit:
|
| 396 |
+
return
|
| 397 |
+
child._search_by_single_name(name, results, limit)
|
| 398 |
+
|
| 399 |
+
# ── find_parent ───────────────────────────────────────────────────────
|
| 400 |
+
|
| 401 |
+
def find_parent(self, name: str | None = None) -> Tag | None:
|
| 402 |
+
"""Walk up the tree and return the first ancestor matching *name*.
|
| 403 |
+
|
| 404 |
+
Args:
|
| 405 |
+
name: Tag name to match. ``None`` returns the immediate parent.
|
| 406 |
+
|
| 407 |
+
Returns:
|
| 408 |
+
The matching ancestor ``Tag``, or ``None``.
|
| 409 |
+
"""
|
| 410 |
+
node = self.parent
|
| 411 |
+
if name is None:
|
| 412 |
+
return node
|
| 413 |
+
while node is not None:
|
| 414 |
+
if node.name == name:
|
| 415 |
+
return node
|
| 416 |
+
node = node.parent
|
| 417 |
+
return None
|
| 418 |
+
|
| 419 |
+
# ── CSS selectors ─────────────────────────────────────────────────────
|
| 420 |
+
|
| 421 |
+
def select(self, css_selector: str) -> list[Tag]:
|
| 422 |
+
"""Return all descendants matching a CSS selector (simple subset).
|
| 423 |
+
|
| 424 |
+
Supported patterns: ``tag``, ``.class``, ``#id``, ``[attr]``,
|
| 425 |
+
``[attr="value"]``, descendant (``a b``), child (``a > b``),
|
| 426 |
+
compound (``div.cls#id``), multiple classes (``div.a.b``).
|
| 427 |
+
|
| 428 |
+
Args:
|
| 429 |
+
css_selector: The CSS selector string.
|
| 430 |
+
|
| 431 |
+
Returns:
|
| 432 |
+
A list of matching ``Tag`` objects.
|
| 433 |
+
"""
|
| 434 |
+
parts = _parse_selector(css_selector)
|
| 435 |
+
candidates: list[Tag] = self._all_descendants()
|
| 436 |
+
return [tag for tag in candidates if _selector_matches(tag, parts)]
|
| 437 |
+
|
| 438 |
+
def select_one(self, css_selector: str) -> Tag | None:
|
| 439 |
+
"""Like ``select``, but return only the first match (or ``None``).
|
| 440 |
+
|
| 441 |
+
Args:
|
| 442 |
+
css_selector: The CSS selector string.
|
| 443 |
+
|
| 444 |
+
Returns:
|
| 445 |
+
The first matching ``Tag``, or ``None``.
|
| 446 |
+
"""
|
| 447 |
+
parts = _parse_selector(css_selector)
|
| 448 |
+
for tag in self._all_descendants():
|
| 449 |
+
if _selector_matches(tag, parts):
|
| 450 |
+
return tag
|
| 451 |
+
return None
|
| 452 |
+
|
| 453 |
+
def _all_descendants(self) -> list[Tag]:
|
| 454 |
+
"""Collect all descendant Tag nodes in document order."""
|
| 455 |
+
result: list[Tag] = []
|
| 456 |
+
self._collect_descendants(result)
|
| 457 |
+
return result
|
| 458 |
+
|
| 459 |
+
def _collect_descendants(self, acc: list[Tag]) -> None:
|
| 460 |
+
for child in self.children:
|
| 461 |
+
if isinstance(child, Tag):
|
| 462 |
+
acc.append(child)
|
| 463 |
+
child._collect_descendants(acc)
|
| 464 |
+
|
| 465 |
+
# ── Serialization ────────────────────────────────────────────────────
|
| 466 |
+
|
| 467 |
+
def to_html(self) -> str:
|
| 468 |
+
"""Serialize this element and its descendants back to an HTML string.
|
| 469 |
+
|
| 470 |
+
Returns:
|
| 471 |
+
The HTML markup for this subtree.
|
| 472 |
+
"""
|
| 473 |
+
parts: list[str] = []
|
| 474 |
+
self._serialize(parts)
|
| 475 |
+
return "".join(parts)
|
| 476 |
+
|
| 477 |
+
def _serialize(self, acc: list[str]) -> None:
|
| 478 |
+
"""Recursively build HTML string pieces into *acc*."""
|
| 479 |
+
# Build opening tag
|
| 480 |
+
attr_parts: list[str] = []
|
| 481 |
+
for k, v in self.attrs.items():
|
| 482 |
+
if isinstance(v, list):
|
| 483 |
+
attr_parts.append(f'{k}="{" ".join(v)}"')
|
| 484 |
+
elif v == "":
|
| 485 |
+
attr_parts.append(k)
|
| 486 |
+
else:
|
| 487 |
+
attr_parts.append(f'{k}="{v}"')
|
| 488 |
+
attrs_str = (" " + " ".join(attr_parts)) if attr_parts else ""
|
| 489 |
+
|
| 490 |
+
if self.name.lower() in SELF_CLOSING_TAGS:
|
| 491 |
+
acc.append(f"<{self.name}{attrs_str}>")
|
| 492 |
+
return
|
| 493 |
+
|
| 494 |
+
acc.append(f"<{self.name}{attrs_str}>")
|
| 495 |
+
for child in self.children:
|
| 496 |
+
if isinstance(child, str):
|
| 497 |
+
acc.append(child)
|
| 498 |
+
else:
|
| 499 |
+
child._serialize(acc)
|
| 500 |
+
acc.append(f"</{self.name}>")
|
| 501 |
+
|
| 502 |
+
def __str__(self) -> str:
|
| 503 |
+
"""Return the HTML serialization of this element."""
|
| 504 |
+
return self.to_html()
|
| 505 |
+
|
| 506 |
+
# ── Repr ──────────────────────────────────────────────────────────────
|
| 507 |
+
|
| 508 |
+
def __repr__(self) -> str:
|
| 509 |
+
attrs_str = ""
|
| 510 |
+
if self.attrs:
|
| 511 |
+
parts = []
|
| 512 |
+
for k, v in self.attrs.items():
|
| 513 |
+
if isinstance(v, list):
|
| 514 |
+
parts.append(f'{k}="{" ".join(v)}"')
|
| 515 |
+
else:
|
| 516 |
+
parts.append(f'{k}="{v}"')
|
| 517 |
+
attrs_str = " " + " ".join(parts)
|
| 518 |
+
return f"<{self.name}{attrs_str}>"
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
# ── Match helpers ─────────────────────────────────────────────────────────────
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def _check_name_match(
|
| 525 |
+
tag: Tag,
|
| 526 |
+
name: str | list[str] | None,
|
| 527 |
+
name_set: frozenset[str] | None = None,
|
| 528 |
+
) -> bool:
|
| 529 |
+
"""Return True if *tag* satisfies the *name* filter."""
|
| 530 |
+
if name is None:
|
| 531 |
+
return True
|
| 532 |
+
if name_set is not None:
|
| 533 |
+
return tag.name in name_set
|
| 534 |
+
if isinstance(name, list):
|
| 535 |
+
return tag.name in name
|
| 536 |
+
return tag.name == name
|
| 537 |
+
|
| 538 |
+
|
| 539 |
+
def _check_attr_filter(tag: Tag, key: str, expected: str | bool) -> bool:
|
| 540 |
+
"""Return True if *tag* satisfies a single attribute filter."""
|
| 541 |
+
actual = tag.attrs.get(key)
|
| 542 |
+
if expected is True:
|
| 543 |
+
return actual is not None
|
| 544 |
+
if expected is False:
|
| 545 |
+
return actual is None
|
| 546 |
+
# Value comparison
|
| 547 |
+
if key == "class":
|
| 548 |
+
return _class_matches(actual, expected)
|
| 549 |
+
if actual is None:
|
| 550 |
+
return False
|
| 551 |
+
if isinstance(actual, list):
|
| 552 |
+
return expected in actual
|
| 553 |
+
return actual == expected
|
| 554 |
+
|
| 555 |
+
|
| 556 |
+
def _matches(
|
| 557 |
+
tag: Tag,
|
| 558 |
+
name: str | list[str] | None,
|
| 559 |
+
name_set: frozenset[str] | None,
|
| 560 |
+
attr_filters: dict[str, str | bool],
|
| 561 |
+
) -> bool:
|
| 562 |
+
"""Check whether *tag* satisfies *name* and *attr_filters*."""
|
| 563 |
+
if not _check_name_match(tag, name, name_set):
|
| 564 |
+
return False
|
| 565 |
+
return all(_check_attr_filter(tag, k, v) for k, v in attr_filters.items())
|
| 566 |
+
|
| 567 |
+
|
| 568 |
+
def _class_matches(actual: str | list[str] | None, expected: str) -> bool:
|
| 569 |
+
"""Return True if *expected* class is present in *actual*.
|
| 570 |
+
|
| 571 |
+
*actual* may be a list (``["a", "b"]``) or ``None``.
|
| 572 |
+
*expected* is a single class name string.
|
| 573 |
+
"""
|
| 574 |
+
if actual is None:
|
| 575 |
+
return False
|
| 576 |
+
if isinstance(actual, list):
|
| 577 |
+
return expected in actual
|
| 578 |
+
return actual == expected
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
# ── CSS selector parser ───────────────────────────────────────────────────────
|
| 582 |
+
|
| 583 |
+
# A parsed selector is a list of *steps*. Each step is
|
| 584 |
+
# ``(combinator, simple_selector_list)`` where combinator is one of
|
| 585 |
+
# ``"descendant"`` or ``"child"`` (the first step has ``"descendant"`` as a
|
| 586 |
+
# dummy). A simple selector is a dict with optional keys ``tag``, ``id``,
|
| 587 |
+
# ``classes``, ``attrs``.
|
| 588 |
+
|
| 589 |
+
_SIMPLE_RE = re.compile(
|
| 590 |
+
r"""
|
| 591 |
+
(?P<tag>[a-zA-Z][a-zA-Z0-9-]*) # tag name
|
| 592 |
+
|(?P<cls>\.[a-zA-Z_-][a-zA-Z0-9_-]*) # .class
|
| 593 |
+
|(?P<id>\#[a-zA-Z_-][a-zA-Z0-9_-]*) # #id
|
| 594 |
+
|(?P<attr>\[[^\]]+\]) # [attr] or [attr="val"]
|
| 595 |
+
|(?P<pseudo>:(?:first-child|last-child|only-child)) # structural pseudo
|
| 596 |
+
|(?P<not>:not\() # :not( — argument parsed separately
|
| 597 |
+
""",
|
| 598 |
+
re.VERBOSE,
|
| 599 |
+
)
|
| 600 |
+
|
| 601 |
+
_ATTR_RE = re.compile(
|
| 602 |
+
r"""
|
| 603 |
+
\[
|
| 604 |
+
\s*(?P<name>[a-zA-Z_-][a-zA-Z0-9_-]*)
|
| 605 |
+
(?:\s*=\s*["']?(?P<value>[^"'\]]*)["']?)?
|
| 606 |
+
\s*\]
|
| 607 |
+
""",
|
| 608 |
+
re.VERBOSE,
|
| 609 |
+
)
|
| 610 |
+
|
| 611 |
+
SelectorStep = tuple[str, dict[str, Any]]
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
_WS = frozenset(" \t\n")
|
| 615 |
+
|
| 616 |
+
|
| 617 |
+
def _skip_whitespace(selector: str, pos: int) -> int:
|
| 618 |
+
"""Advance *pos* past any whitespace characters."""
|
| 619 |
+
while pos < len(selector) and selector[pos] in _WS:
|
| 620 |
+
pos += 1
|
| 621 |
+
return pos
|
| 622 |
+
|
| 623 |
+
|
| 624 |
+
def _tokenize_whitespace(
|
| 625 |
+
selector: str, pos: int, tokens: list[str | dict[str, Any]]
|
| 626 |
+
) -> int:
|
| 627 |
+
"""Handle whitespace at *pos*, emitting a combinator token if needed.
|
| 628 |
+
|
| 629 |
+
Returns the updated position.
|
| 630 |
+
"""
|
| 631 |
+
rest = selector[pos:].lstrip()
|
| 632 |
+
if rest.startswith(">"):
|
| 633 |
+
tokens.append(">")
|
| 634 |
+
pos = selector.index(">", pos) + 1
|
| 635 |
+
return _skip_whitespace(selector, pos)
|
| 636 |
+
# Space combinator only if the previous token is not already a combinator
|
| 637 |
+
if tokens and tokens[-1] not in (" ", ">"):
|
| 638 |
+
tokens.append(" ")
|
| 639 |
+
return pos + 1
|
| 640 |
+
|
| 641 |
+
|
| 642 |
+
def _apply_regex_match(m: re.Match[str], compound: dict[str, Any]) -> None:
|
| 643 |
+
"""Populate *compound* dict from a single ``_SIMPLE_RE`` match."""
|
| 644 |
+
if m.group("tag"):
|
| 645 |
+
compound["tag"] = m.group("tag")
|
| 646 |
+
elif m.group("cls"):
|
| 647 |
+
compound.setdefault("classes", []).append(m.group("cls")[1:])
|
| 648 |
+
elif m.group("id"):
|
| 649 |
+
compound["id"] = m.group("id")[1:]
|
| 650 |
+
elif m.group("attr"):
|
| 651 |
+
am = _ATTR_RE.match(m.group("attr"))
|
| 652 |
+
if am:
|
| 653 |
+
compound.setdefault("attrs", []).append(
|
| 654 |
+
(am.group("name"), am.group("value"))
|
| 655 |
+
)
|
| 656 |
+
elif m.group("pseudo"):
|
| 657 |
+
compound.setdefault("pseudos", []).append(m.group("pseudo")[1:])
|
| 658 |
+
|
| 659 |
+
|
| 660 |
+
def _parse_not_argument(selector: str, pos: int) -> tuple[dict[str, Any], int]:
|
| 661 |
+
"""Parse the simple selector inside ``:not(...)``, return (inner, new_pos).
|
| 662 |
+
|
| 663 |
+
*pos* should point right after the opening ``(``. The closing ``)`` is
|
| 664 |
+
consumed and *new_pos* points past it.
|
| 665 |
+
"""
|
| 666 |
+
inner, pos = _parse_compound(selector, pos)
|
| 667 |
+
if inner is None:
|
| 668 |
+
inner = {}
|
| 669 |
+
pos = _skip_whitespace(selector, pos)
|
| 670 |
+
if pos < len(selector) and selector[pos] == ")":
|
| 671 |
+
pos += 1
|
| 672 |
+
return inner, pos
|
| 673 |
+
|
| 674 |
+
|
| 675 |
+
def _parse_compound(selector: str, pos: int) -> tuple[dict[str, Any] | None, int]:
|
| 676 |
+
"""Parse a compound simple selector starting at *pos*.
|
| 677 |
+
|
| 678 |
+
Returns ``(compound_dict_or_None, new_pos)``.
|
| 679 |
+
"""
|
| 680 |
+
compound: dict[str, Any] = {}
|
| 681 |
+
matched_any = False
|
| 682 |
+
while pos < len(selector):
|
| 683 |
+
m = _SIMPLE_RE.match(selector, pos)
|
| 684 |
+
if m is None:
|
| 685 |
+
break
|
| 686 |
+
matched_any = True
|
| 687 |
+
if m.group("not"):
|
| 688 |
+
# :not( matched — parse the inner selector and closing paren
|
| 689 |
+
pos = m.end()
|
| 690 |
+
inner, pos = _parse_not_argument(selector, pos)
|
| 691 |
+
compound.setdefault("nots", []).append(inner)
|
| 692 |
+
else:
|
| 693 |
+
_apply_regex_match(m, compound)
|
| 694 |
+
pos = m.end()
|
| 695 |
+
return (compound if matched_any else None), pos
|
| 696 |
+
|
| 697 |
+
|
| 698 |
+
def _tokens_to_steps(tokens: list[str | dict[str, Any]]) -> list[SelectorStep]:
|
| 699 |
+
"""Convert a flat token list into ``(combinator, simple)`` step list."""
|
| 700 |
+
steps: list[SelectorStep] = []
|
| 701 |
+
combinator = "descendant"
|
| 702 |
+
for tok in tokens:
|
| 703 |
+
if tok == " ":
|
| 704 |
+
combinator = "descendant"
|
| 705 |
+
elif tok == ">":
|
| 706 |
+
combinator = "child"
|
| 707 |
+
else:
|
| 708 |
+
assert isinstance(tok, dict)
|
| 709 |
+
steps.append((combinator, tok))
|
| 710 |
+
combinator = "descendant"
|
| 711 |
+
return steps
|
| 712 |
+
|
| 713 |
+
|
| 714 |
+
def _parse_selector(selector: str) -> list[SelectorStep]:
|
| 715 |
+
"""Parse a simple CSS selector string into step list.
|
| 716 |
+
|
| 717 |
+
Returns a list of ``(combinator, simple)`` tuples.
|
| 718 |
+
"""
|
| 719 |
+
tokens: list[str | dict[str, Any]] = []
|
| 720 |
+
pos = 0
|
| 721 |
+
selector = selector.strip()
|
| 722 |
+
|
| 723 |
+
while pos < len(selector):
|
| 724 |
+
if selector[pos] in _WS:
|
| 725 |
+
pos = _tokenize_whitespace(selector, pos, tokens)
|
| 726 |
+
continue
|
| 727 |
+
|
| 728 |
+
if selector[pos] == ">":
|
| 729 |
+
tokens.append(">")
|
| 730 |
+
pos = _skip_whitespace(selector, pos + 1)
|
| 731 |
+
continue
|
| 732 |
+
|
| 733 |
+
compound, pos = _parse_compound(selector, pos)
|
| 734 |
+
if compound is not None:
|
| 735 |
+
tokens.append(compound)
|
| 736 |
+
else:
|
| 737 |
+
pos += 1 # skip unknown character to avoid infinite loop
|
| 738 |
+
|
| 739 |
+
return _tokens_to_steps(tokens)
|
| 740 |
+
|
| 741 |
+
|
| 742 |
+
def _classes_match(tag: Tag, required_classes: list[str]) -> bool:
|
| 743 |
+
"""Return True if *tag* has all *required_classes*."""
|
| 744 |
+
tag_classes = tag.attrs.get("class")
|
| 745 |
+
if tag_classes is None:
|
| 746 |
+
return False
|
| 747 |
+
if isinstance(tag_classes, str):
|
| 748 |
+
tag_classes = [tag_classes]
|
| 749 |
+
return all(cls in tag_classes for cls in required_classes)
|
| 750 |
+
|
| 751 |
+
|
| 752 |
+
def _attr_value_matches(actual: str | list[str], expected: str) -> bool:
|
| 753 |
+
"""Return True if *actual* attribute value equals *expected*."""
|
| 754 |
+
if isinstance(actual, list):
|
| 755 |
+
return expected in actual
|
| 756 |
+
return actual == expected
|
| 757 |
+
|
| 758 |
+
|
| 759 |
+
def _selector_attrs_match(tag: Tag, attrs: list[tuple[str, str | None]]) -> bool:
|
| 760 |
+
"""Return True if *tag* satisfies all attribute constraints from a selector."""
|
| 761 |
+
for attr_name, attr_val in attrs:
|
| 762 |
+
actual = tag.attrs.get(attr_name)
|
| 763 |
+
if actual is None:
|
| 764 |
+
return False
|
| 765 |
+
if attr_val is not None and not _attr_value_matches(actual, attr_val):
|
| 766 |
+
return False
|
| 767 |
+
return True
|
| 768 |
+
|
| 769 |
+
|
| 770 |
+
def _element_siblings(tag: Tag) -> list[Tag]:
|
| 771 |
+
"""Return element (non-text) children of *tag*'s parent."""
|
| 772 |
+
if tag.parent is None:
|
| 773 |
+
return [tag]
|
| 774 |
+
return [c for c in tag.parent.children if isinstance(c, Tag)]
|
| 775 |
+
|
| 776 |
+
|
| 777 |
+
def _pseudo_matches(tag: Tag, pseudo: str) -> bool:
|
| 778 |
+
"""Return True if *tag* satisfies the structural pseudo-class *pseudo*."""
|
| 779 |
+
siblings = _element_siblings(tag)
|
| 780 |
+
if pseudo == "first-child":
|
| 781 |
+
return siblings[0] is tag
|
| 782 |
+
if pseudo == "last-child":
|
| 783 |
+
return siblings[-1] is tag
|
| 784 |
+
if pseudo == "only-child":
|
| 785 |
+
return len(siblings) == 1
|
| 786 |
+
return False # pragma: no cover
|
| 787 |
+
|
| 788 |
+
|
| 789 |
+
def _simple_matches(tag: Tag, simple: dict[str, Any]) -> bool:
|
| 790 |
+
"""Check if *tag* matches a single compound simple selector."""
|
| 791 |
+
if "tag" in simple and tag.name != simple["tag"]:
|
| 792 |
+
return False
|
| 793 |
+
if "id" in simple and tag.attrs.get("id") != simple["id"]:
|
| 794 |
+
return False
|
| 795 |
+
if "classes" in simple and not _classes_match(tag, simple["classes"]):
|
| 796 |
+
return False
|
| 797 |
+
if "attrs" in simple and not _selector_attrs_match(tag, simple["attrs"]):
|
| 798 |
+
return False
|
| 799 |
+
if "pseudos" in simple:
|
| 800 |
+
for pseudo in simple["pseudos"]:
|
| 801 |
+
if not _pseudo_matches(tag, pseudo):
|
| 802 |
+
return False
|
| 803 |
+
if "nots" in simple:
|
| 804 |
+
for inner in simple["nots"]:
|
| 805 |
+
if _simple_matches(tag, inner):
|
| 806 |
+
return False
|
| 807 |
+
return True
|
| 808 |
+
|
| 809 |
+
|
| 810 |
+
def _selector_matches(tag: Tag, steps: list[SelectorStep]) -> bool:
|
| 811 |
+
"""Return ``True`` if *tag* matches the full parsed selector."""
|
| 812 |
+
if not steps:
|
| 813 |
+
return True
|
| 814 |
+
|
| 815 |
+
# The last step must match the tag itself.
|
| 816 |
+
combinator, simple = steps[-1]
|
| 817 |
+
if not _simple_matches(tag, simple):
|
| 818 |
+
return False
|
| 819 |
+
|
| 820 |
+
if len(steps) == 1:
|
| 821 |
+
return True
|
| 822 |
+
|
| 823 |
+
# Walk remaining steps backwards up the tree.
|
| 824 |
+
remaining = steps[:-1]
|
| 825 |
+
return _ancestor_matches(tag, remaining)
|
| 826 |
+
|
| 827 |
+
|
| 828 |
+
def _ancestor_matches(tag: Tag, steps: list[SelectorStep]) -> bool:
|
| 829 |
+
"""Verify that ancestors of *tag* satisfy the remaining selector steps."""
|
| 830 |
+
if not steps:
|
| 831 |
+
return True
|
| 832 |
+
|
| 833 |
+
combinator, simple = steps[-1]
|
| 834 |
+
rest = steps[:-1]
|
| 835 |
+
|
| 836 |
+
if combinator == "child":
|
| 837 |
+
parent = tag.parent
|
| 838 |
+
if parent is None or not _simple_matches(parent, simple):
|
| 839 |
+
return False
|
| 840 |
+
return _ancestor_matches(parent, rest)
|
| 841 |
+
else:
|
| 842 |
+
# descendant — walk up until we find a match
|
| 843 |
+
node = tag.parent
|
| 844 |
+
while node is not None:
|
| 845 |
+
if _simple_matches(node, simple):
|
| 846 |
+
if _ancestor_matches(node, rest):
|
| 847 |
+
return True
|
| 848 |
+
node = node.parent
|
| 849 |
+
return False
|
| 850 |
+
|
| 851 |
+
|
| 852 |
+
# ── HTML parser ───────────────────────────────────────────────────────────────
|
| 853 |
+
|
| 854 |
+
|
| 855 |
+
class _TreeBuilder(HTMLParser):
|
| 856 |
+
"""Build a ``Tag`` tree from HTML markup.
|
| 857 |
+
|
| 858 |
+
Args:
|
| 859 |
+
skip_tags: Optional frozenset of tag names to omit from the tree.
|
| 860 |
+
When a skipped tag is encountered, it and all its descendants
|
| 861 |
+
(including text) are silently discarded.
|
| 862 |
+
"""
|
| 863 |
+
|
| 864 |
+
def __init__(self, skip_tags: frozenset[str] | None = None) -> None:
|
| 865 |
+
super().__init__(convert_charrefs=True)
|
| 866 |
+
self.root = Tag("[document]")
|
| 867 |
+
self.current: Tag = self.root
|
| 868 |
+
self._skip_tags: frozenset[str] = skip_tags or frozenset()
|
| 869 |
+
# Depth counter for nested skipped tags (> 0 means discarding).
|
| 870 |
+
self._skip_depth: int = 0
|
| 871 |
+
|
| 872 |
+
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
| 873 |
+
# If we're inside a skipped subtree, track nesting depth only.
|
| 874 |
+
if self._skip_depth > 0:
|
| 875 |
+
if tag not in SELF_CLOSING_TAGS:
|
| 876 |
+
self._skip_depth += 1
|
| 877 |
+
return
|
| 878 |
+
|
| 879 |
+
# Check if this tag should be skipped.
|
| 880 |
+
if tag in self._skip_tags:
|
| 881 |
+
if tag not in SELF_CLOSING_TAGS:
|
| 882 |
+
self._skip_depth = 1
|
| 883 |
+
return
|
| 884 |
+
|
| 885 |
+
attr_dict: dict[str, str | list[str]] = {}
|
| 886 |
+
for key, value in attrs:
|
| 887 |
+
if value is None:
|
| 888 |
+
value = ""
|
| 889 |
+
if key == "class":
|
| 890 |
+
attr_dict["class"] = value.split()
|
| 891 |
+
else:
|
| 892 |
+
attr_dict[key] = value
|
| 893 |
+
|
| 894 |
+
node = Tag(tag, attr_dict, parent=self.current)
|
| 895 |
+
self.current.children.append(node)
|
| 896 |
+
|
| 897 |
+
if tag not in SELF_CLOSING_TAGS:
|
| 898 |
+
self.current = node
|
| 899 |
+
|
| 900 |
+
def handle_endtag(self, tag: str) -> None:
|
| 901 |
+
# If we're inside a skipped subtree, decrement depth.
|
| 902 |
+
if self._skip_depth > 0:
|
| 903 |
+
self._skip_depth -= 1
|
| 904 |
+
return
|
| 905 |
+
|
| 906 |
+
# Walk up to find the matching open tag (tolerates malformed HTML)
|
| 907 |
+
node = self.current
|
| 908 |
+
while node is not None and node.name != tag:
|
| 909 |
+
node = node.parent
|
| 910 |
+
if node is not None and node.parent is not None:
|
| 911 |
+
self.current = node.parent
|
| 912 |
+
# If no matching open tag found, do nothing (malformed HTML tolerance)
|
| 913 |
+
|
| 914 |
+
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
| 915 |
+
# Explicit self-closing tag like <br/>
|
| 916 |
+
if self._skip_depth > 0 or tag in self._skip_tags:
|
| 917 |
+
return
|
| 918 |
+
self.handle_starttag(tag, attrs)
|
| 919 |
+
# Don't descend into it — it has no children. If handle_starttag
|
| 920 |
+
# pushed current down, pop back up.
|
| 921 |
+
if self.current.name == tag:
|
| 922 |
+
if self.current.parent is not None:
|
| 923 |
+
self.current = self.current.parent
|
| 924 |
+
|
| 925 |
+
def handle_data(self, data: str) -> None:
|
| 926 |
+
if self._skip_depth > 0:
|
| 927 |
+
return
|
| 928 |
+
self.current.children.append(data)
|
| 929 |
+
|
| 930 |
+
def handle_comment(self, data: str) -> None:
|
| 931 |
+
# Comments are silently dropped (matching BS4 default).
|
| 932 |
+
pass
|
| 933 |
+
|
| 934 |
+
|
| 935 |
+
# ── Soup (document root) ─────────────────────────────────────────────────────
|
| 936 |
+
|
| 937 |
+
|
| 938 |
+
class Soup(Tag):
|
| 939 |
+
"""Parse an HTML document and provide a BeautifulSoup-like API.
|
| 940 |
+
|
| 941 |
+
Args:
|
| 942 |
+
markup: The HTML string to parse.
|
| 943 |
+
parser: Ignored (present only for API compatibility with BS4).
|
| 944 |
+
Only ``"html.parser"`` is supported.
|
| 945 |
+
skip_tags: Optional frozenset of tag names to omit during parsing.
|
| 946 |
+
Skipped tags and all their descendants are silently discarded,
|
| 947 |
+
which can significantly speed up parsing of pages with many
|
| 948 |
+
``<script>`` or ``<style>`` blocks.
|
| 949 |
+
|
| 950 |
+
Example::
|
| 951 |
+
|
| 952 |
+
soup = Soup("<p>Hello <b>world</b></p>")
|
| 953 |
+
print(soup.find("b").text)
|
| 954 |
+
# world
|
| 955 |
+
"""
|
| 956 |
+
|
| 957 |
+
def __init__(
|
| 958 |
+
self,
|
| 959 |
+
markup: str,
|
| 960 |
+
parser: str = "html.parser",
|
| 961 |
+
skip_tags: frozenset[str] | None = None,
|
| 962 |
+
) -> None:
|
| 963 |
+
super().__init__("[document]")
|
| 964 |
+
builder = _TreeBuilder(skip_tags=skip_tags)
|
| 965 |
+
builder.feed(markup)
|
| 966 |
+
# Adopt the root's children as our own.
|
| 967 |
+
self.children = builder.root.children
|
| 968 |
+
for child in self.children:
|
| 969 |
+
if isinstance(child, Tag):
|
| 970 |
+
child.parent = self
|
| 971 |
+
|
| 972 |
+
def new_tag(
|
| 973 |
+
self, name: str, attrs: dict[str, str | list[str]] | None = None
|
| 974 |
+
) -> Tag:
|
| 975 |
+
"""Create a new detached ``Tag`` (not yet in the tree).
|
| 976 |
+
|
| 977 |
+
Args:
|
| 978 |
+
name: Tag name (e.g. ``"p"``).
|
| 979 |
+
attrs: Optional attribute dictionary.
|
| 980 |
+
|
| 981 |
+
Returns:
|
| 982 |
+
A new ``Tag`` instance with no parent.
|
| 983 |
+
"""
|
| 984 |
+
return Tag(name, attrs)
|
| 985 |
+
|
| 986 |
+
def to_html(self) -> str:
|
| 987 |
+
"""Serialize the entire document back to an HTML string.
|
| 988 |
+
|
| 989 |
+
Returns:
|
| 990 |
+
The HTML markup for the whole document.
|
| 991 |
+
"""
|
| 992 |
+
parts: list[str] = []
|
| 993 |
+
for child in self.children:
|
| 994 |
+
if isinstance(child, str):
|
| 995 |
+
parts.append(child)
|
| 996 |
+
else:
|
| 997 |
+
child._serialize(parts)
|
| 998 |
+
return "".join(parts)
|
|
@@ -0,0 +1,888 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.3.0"
|
| 3 |
+
# deps = []
|
| 4 |
+
# tier = "medium"
|
| 5 |
+
# category = "devtools"
|
| 6 |
+
# note = "Install/update via `zerodep add structlog`"
|
| 7 |
+
# ///
|
| 8 |
+
|
| 9 |
+
"""Zero-dependency structured logging with pretty console output.
|
| 10 |
+
|
| 11 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 12 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 13 |
+
|
| 14 |
+
Structured logging library inspired by structlog, with a loguru-style
|
| 15 |
+
colored console renderer. Provides bound loggers with context propagation,
|
| 16 |
+
a processor pipeline, and multiple output renderers (console, JSON, key-value).
|
| 17 |
+
|
| 18 |
+
Quick start (pretty console output, zero config)::
|
| 19 |
+
|
| 20 |
+
from structlog import get_logger
|
| 21 |
+
logger = get_logger()
|
| 22 |
+
logger.info("server started", host="0.0.0.0", port=8080)
|
| 23 |
+
|
| 24 |
+
Bound logger (context propagation)::
|
| 25 |
+
|
| 26 |
+
log = get_logger().bind(request_id="abc-123")
|
| 27 |
+
log.info("handling request")
|
| 28 |
+
log = log.bind(user_id=42)
|
| 29 |
+
log.info("authenticated")
|
| 30 |
+
|
| 31 |
+
One-call setup with stdlib integration::
|
| 32 |
+
|
| 33 |
+
from structlog import setup_logging
|
| 34 |
+
logger = setup_logging(level="DEBUG", renderer="json")
|
| 35 |
+
logger.info("structured", key="value")
|
| 36 |
+
|
| 37 |
+
Custom processor pipeline::
|
| 38 |
+
|
| 39 |
+
from structlog import configure, add_log_level, TimeStamper, JSONRenderer
|
| 40 |
+
configure(processors=[add_log_level, TimeStamper(), JSONRenderer()])
|
| 41 |
+
"""
|
| 42 |
+
|
| 43 |
+
from __future__ import annotations
|
| 44 |
+
|
| 45 |
+
import dataclasses
|
| 46 |
+
import datetime
|
| 47 |
+
import json as _json
|
| 48 |
+
import logging
|
| 49 |
+
import os
|
| 50 |
+
import sys
|
| 51 |
+
import traceback
|
| 52 |
+
from typing import IO, Any, Callable
|
| 53 |
+
|
| 54 |
+
__all__ = [
|
| 55 |
+
# Type aliases
|
| 56 |
+
"EventDict",
|
| 57 |
+
"Processor",
|
| 58 |
+
"LoggerFactory",
|
| 59 |
+
# Exceptions
|
| 60 |
+
"DropEvent",
|
| 61 |
+
# Logger factories
|
| 62 |
+
"PrintLogger",
|
| 63 |
+
"PrintLoggerFactory",
|
| 64 |
+
"StdlibLoggerFactory",
|
| 65 |
+
# Processors
|
| 66 |
+
"add_log_level",
|
| 67 |
+
"add_logger_name",
|
| 68 |
+
"TimeStamper",
|
| 69 |
+
"format_exc_info",
|
| 70 |
+
# Renderers
|
| 71 |
+
"KeyValueRenderer",
|
| 72 |
+
"JSONRenderer",
|
| 73 |
+
"ConsoleRenderer",
|
| 74 |
+
# Bound logger
|
| 75 |
+
"BoundLogger",
|
| 76 |
+
# Configuration
|
| 77 |
+
"configure",
|
| 78 |
+
"reset_defaults",
|
| 79 |
+
"get_config",
|
| 80 |
+
"get_logger",
|
| 81 |
+
"wrap_logger",
|
| 82 |
+
# Setup helper
|
| 83 |
+
"setup_logging",
|
| 84 |
+
# Utilities
|
| 85 |
+
"truncate_string",
|
| 86 |
+
"truncate_base64",
|
| 87 |
+
]
|
| 88 |
+
|
| 89 |
+
# ── Type Aliases ─────────────────────────────────────────────────────────────
|
| 90 |
+
|
| 91 |
+
EventDict = dict[str, Any]
|
| 92 |
+
"""A log event dictionary passed through the processor pipeline."""
|
| 93 |
+
|
| 94 |
+
Processor = Callable[[Any, str, EventDict], EventDict | str]
|
| 95 |
+
"""Signature: (logger, method_name, event_dict) -> event_dict or rendered str."""
|
| 96 |
+
|
| 97 |
+
LoggerFactory = Callable[..., Any]
|
| 98 |
+
"""Callable that creates an underlying logger instance."""
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# ── Exceptions ───────────────────────────────────────────────────────────────
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
class DropEvent(Exception):
|
| 105 |
+
"""Raise inside a processor to silently discard the current log event."""
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
# ── Level Mapping ────────────────────────────────────────────────────────────
|
| 109 |
+
|
| 110 |
+
_NAME_TO_LEVEL: dict[str, int] = {
|
| 111 |
+
"debug": logging.DEBUG,
|
| 112 |
+
"info": logging.INFO,
|
| 113 |
+
"warning": logging.WARNING,
|
| 114 |
+
"warn": logging.WARNING,
|
| 115 |
+
"error": logging.ERROR,
|
| 116 |
+
"critical": logging.CRITICAL,
|
| 117 |
+
"fatal": logging.CRITICAL,
|
| 118 |
+
"exception": logging.ERROR,
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
_LEVEL_TO_NAME: dict[int, str] = {
|
| 122 |
+
logging.DEBUG: "debug",
|
| 123 |
+
logging.INFO: "info",
|
| 124 |
+
logging.WARNING: "warning",
|
| 125 |
+
logging.ERROR: "error",
|
| 126 |
+
logging.CRITICAL: "critical",
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
# ── ANSI Colors ──────────────────────────────────────────────────────────────
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
class _Colors:
|
| 134 |
+
"""ANSI escape codes for terminal colorization."""
|
| 135 |
+
|
| 136 |
+
RESET = "\033[0m"
|
| 137 |
+
BOLD = "\033[1m"
|
| 138 |
+
DIM = "\033[2m"
|
| 139 |
+
ITALIC = "\033[3m"
|
| 140 |
+
UNDERLINE = "\033[4m"
|
| 141 |
+
STRIKETHROUGH = "\033[9m"
|
| 142 |
+
|
| 143 |
+
BLACK = "\033[30m"
|
| 144 |
+
RED = "\033[31m"
|
| 145 |
+
GREEN = "\033[32m"
|
| 146 |
+
YELLOW = "\033[33m"
|
| 147 |
+
BLUE = "\033[34m"
|
| 148 |
+
MAGENTA = "\033[35m"
|
| 149 |
+
CYAN = "\033[36m"
|
| 150 |
+
WHITE = "\033[37m"
|
| 151 |
+
|
| 152 |
+
BRIGHT_BLACK = "\033[90m"
|
| 153 |
+
BRIGHT_RED = "\033[91m"
|
| 154 |
+
BRIGHT_GREEN = "\033[92m"
|
| 155 |
+
BRIGHT_YELLOW = "\033[93m"
|
| 156 |
+
BRIGHT_BLUE = "\033[94m"
|
| 157 |
+
BRIGHT_MAGENTA = "\033[95m"
|
| 158 |
+
BRIGHT_CYAN = "\033[96m"
|
| 159 |
+
BRIGHT_WHITE = "\033[97m"
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
_LEVEL_STYLES: dict[str, str] = {
|
| 163 |
+
"debug": _Colors.BLUE,
|
| 164 |
+
"info": _Colors.BRIGHT_WHITE,
|
| 165 |
+
"warning": _Colors.YELLOW,
|
| 166 |
+
"warn": _Colors.YELLOW,
|
| 167 |
+
"error": _Colors.RED,
|
| 168 |
+
"critical": _Colors.BRIGHT_RED + _Colors.BOLD,
|
| 169 |
+
"fatal": _Colors.BRIGHT_RED + _Colors.BOLD,
|
| 170 |
+
"exception": _Colors.RED,
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
_LEVEL_LABEL_STYLES: dict[str, str] = {
|
| 174 |
+
"debug": _Colors.CYAN,
|
| 175 |
+
"info": _Colors.GREEN,
|
| 176 |
+
"warning": _Colors.YELLOW,
|
| 177 |
+
"warn": _Colors.YELLOW,
|
| 178 |
+
"error": _Colors.RED,
|
| 179 |
+
"critical": _Colors.BRIGHT_RED + _Colors.BOLD,
|
| 180 |
+
"fatal": _Colors.BRIGHT_RED + _Colors.BOLD,
|
| 181 |
+
"exception": _Colors.RED,
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
_LEVEL_LABELS: dict[str, str] = {
|
| 185 |
+
"debug": "DEBUG ",
|
| 186 |
+
"info": "INFO ",
|
| 187 |
+
"warning": "WARNING ",
|
| 188 |
+
"warn": "WARNING ",
|
| 189 |
+
"error": "ERROR ",
|
| 190 |
+
"critical": "CRITICAL",
|
| 191 |
+
"fatal": "FATAL ",
|
| 192 |
+
"exception": "ERROR ",
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def _supports_color(stream: Any = None) -> bool:
|
| 197 |
+
"""Check if the output stream supports ANSI color codes.
|
| 198 |
+
|
| 199 |
+
Respects the ``FORCE_COLOR`` and ``NO_COLOR`` environment variables
|
| 200 |
+
(see https://force-color.org/ and https://no-color.org/).
|
| 201 |
+
|
| 202 |
+
Args:
|
| 203 |
+
stream: The output stream to check. Defaults to ``sys.stderr``.
|
| 204 |
+
|
| 205 |
+
Returns:
|
| 206 |
+
True if the stream is a color-capable terminal.
|
| 207 |
+
"""
|
| 208 |
+
if os.environ.get("FORCE_COLOR"):
|
| 209 |
+
return True
|
| 210 |
+
if os.environ.get("NO_COLOR"):
|
| 211 |
+
return False
|
| 212 |
+
s = stream or sys.stderr
|
| 213 |
+
if not hasattr(s, "isatty") or not s.isatty():
|
| 214 |
+
return False
|
| 215 |
+
if os.environ.get("TERM", "") == "dumb":
|
| 216 |
+
return False
|
| 217 |
+
return True
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
# ── Logger Factories ─────────────────────────────────────────────────────────
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
class PrintLogger:
|
| 224 |
+
"""Minimal logger that writes to a file handle via ``print()``.
|
| 225 |
+
|
| 226 |
+
This is the default underlying logger. It has no filtering, no
|
| 227 |
+
handler chain -- just ``print()`` to the configured stream.
|
| 228 |
+
|
| 229 |
+
Args:
|
| 230 |
+
file: Output stream. Defaults to ``sys.stderr``.
|
| 231 |
+
"""
|
| 232 |
+
|
| 233 |
+
def __init__(self, file: IO[str] | None = None) -> None:
|
| 234 |
+
self._file = file or sys.stderr
|
| 235 |
+
self._write = self._file.write
|
| 236 |
+
self._flush = self._file.flush
|
| 237 |
+
|
| 238 |
+
def msg(self, message: str) -> None:
|
| 239 |
+
"""Write *message* followed by a newline."""
|
| 240 |
+
self._write(message + "\n")
|
| 241 |
+
self._flush()
|
| 242 |
+
|
| 243 |
+
debug = info = warning = warn = error = critical = fatal = msg
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
class PrintLoggerFactory:
|
| 247 |
+
"""Factory that creates :class:`PrintLogger` instances.
|
| 248 |
+
|
| 249 |
+
Args:
|
| 250 |
+
file: Output stream passed to each ``PrintLogger``.
|
| 251 |
+
"""
|
| 252 |
+
|
| 253 |
+
def __init__(self, file: IO[str] | None = None) -> None:
|
| 254 |
+
self._file = file
|
| 255 |
+
|
| 256 |
+
def __call__(self, *args: Any) -> PrintLogger:
|
| 257 |
+
return PrintLogger(file=self._file)
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
class StdlibLoggerFactory:
|
| 261 |
+
"""Factory that returns a :class:`logging.Logger` from the stdlib.
|
| 262 |
+
|
| 263 |
+
Args:
|
| 264 |
+
name: Logger name passed to ``logging.getLogger()``.
|
| 265 |
+
If *None*, uses the root logger.
|
| 266 |
+
"""
|
| 267 |
+
|
| 268 |
+
def __init__(self, name: str | None = None) -> None:
|
| 269 |
+
self._name = name
|
| 270 |
+
|
| 271 |
+
def __call__(self, *args: Any) -> logging.Logger:
|
| 272 |
+
if args:
|
| 273 |
+
return logging.getLogger(args[0])
|
| 274 |
+
return logging.getLogger(self._name)
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
# ── Built-in Processors ──────────────────────────────────────────────────────
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def add_log_level(logger: Any, method_name: str, event_dict: EventDict) -> EventDict:
|
| 281 |
+
"""Add ``level`` key derived from the log method name.
|
| 282 |
+
|
| 283 |
+
Example:
|
| 284 |
+
``logger.info(...)`` -> ``event_dict["level"] = "info"``
|
| 285 |
+
"""
|
| 286 |
+
event_dict["level"] = method_name
|
| 287 |
+
return event_dict
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def add_logger_name(logger: Any, method_name: str, event_dict: EventDict) -> EventDict:
|
| 291 |
+
"""Add ``logger`` key from the underlying logger's name."""
|
| 292 |
+
name = getattr(logger, "name", None) or ""
|
| 293 |
+
event_dict["logger"] = name
|
| 294 |
+
return event_dict
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
class TimeStamper:
|
| 298 |
+
"""Processor that adds a timestamp to the event dict.
|
| 299 |
+
|
| 300 |
+
Args:
|
| 301 |
+
fmt: Timestamp format. ``"iso"`` for ISO 8601, ``None`` for a
|
| 302 |
+
UNIX float, or a :func:`~datetime.datetime.strftime` format
|
| 303 |
+
string. Defaults to ``"iso"``.
|
| 304 |
+
utc: If *True*, use UTC; otherwise local time.
|
| 305 |
+
key: Dict key for the timestamp. Defaults to ``"timestamp"``.
|
| 306 |
+
"""
|
| 307 |
+
|
| 308 |
+
__slots__ = ("_fmt", "_utc", "_key")
|
| 309 |
+
|
| 310 |
+
def __init__(
|
| 311 |
+
self,
|
| 312 |
+
fmt: str | None = "iso",
|
| 313 |
+
utc: bool = True,
|
| 314 |
+
key: str = "timestamp",
|
| 315 |
+
) -> None:
|
| 316 |
+
self._fmt = fmt
|
| 317 |
+
self._utc = utc
|
| 318 |
+
self._key = key
|
| 319 |
+
|
| 320 |
+
def __call__(
|
| 321 |
+
self, logger: Any, method_name: str, event_dict: EventDict
|
| 322 |
+
) -> EventDict:
|
| 323 |
+
now = (
|
| 324 |
+
datetime.datetime.now(datetime.timezone.utc)
|
| 325 |
+
if self._utc
|
| 326 |
+
else datetime.datetime.now()
|
| 327 |
+
)
|
| 328 |
+
if self._fmt is None:
|
| 329 |
+
event_dict[self._key] = now.timestamp()
|
| 330 |
+
elif self._fmt == "iso":
|
| 331 |
+
event_dict[self._key] = now.isoformat()
|
| 332 |
+
else:
|
| 333 |
+
event_dict[self._key] = now.strftime(self._fmt)
|
| 334 |
+
return event_dict
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
def format_exc_info(logger: Any, method_name: str, event_dict: EventDict) -> EventDict:
|
| 338 |
+
"""Replace ``exc_info`` with a formatted ``exception`` string.
|
| 339 |
+
|
| 340 |
+
If ``exc_info`` is *True*, captures the current exception via
|
| 341 |
+
:func:`sys.exc_info`. If it is an exception tuple, formats it
|
| 342 |
+
directly. The ``exc_info`` key is removed and replaced with
|
| 343 |
+
``exception``.
|
| 344 |
+
"""
|
| 345 |
+
exc_info = event_dict.pop("exc_info", None)
|
| 346 |
+
if exc_info is None or exc_info is False:
|
| 347 |
+
return event_dict
|
| 348 |
+
|
| 349 |
+
if exc_info is True:
|
| 350 |
+
exc_info = sys.exc_info()
|
| 351 |
+
|
| 352 |
+
if isinstance(exc_info, BaseException):
|
| 353 |
+
exc_info = (type(exc_info), exc_info, exc_info.__traceback__)
|
| 354 |
+
|
| 355 |
+
if isinstance(exc_info, tuple) and exc_info[0] is not None:
|
| 356 |
+
event_dict["exception"] = "".join(
|
| 357 |
+
traceback.format_exception(*exc_info)
|
| 358 |
+
).rstrip()
|
| 359 |
+
|
| 360 |
+
return event_dict
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
# ── Renderers (final processors) ─────────────────────────────────────────────
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
class KeyValueRenderer:
|
| 367 |
+
"""Render the event dict as ``key=value`` pairs.
|
| 368 |
+
|
| 369 |
+
Args:
|
| 370 |
+
key_order: Keys to render first, in this order.
|
| 371 |
+
sort_keys: Sort remaining keys alphabetically.
|
| 372 |
+
drop_missing: Skip *key_order* keys that are absent from the dict.
|
| 373 |
+
"""
|
| 374 |
+
|
| 375 |
+
__slots__ = ("_key_order", "_sort_keys", "_drop_missing")
|
| 376 |
+
|
| 377 |
+
def __init__(
|
| 378 |
+
self,
|
| 379 |
+
key_order: list[str] | None = None,
|
| 380 |
+
sort_keys: bool = False,
|
| 381 |
+
drop_missing: bool = True,
|
| 382 |
+
) -> None:
|
| 383 |
+
self._key_order = key_order or []
|
| 384 |
+
self._sort_keys = sort_keys
|
| 385 |
+
self._drop_missing = drop_missing
|
| 386 |
+
|
| 387 |
+
def __call__(self, logger: Any, method_name: str, event_dict: EventDict) -> str:
|
| 388 |
+
ordered: list[tuple[str, Any]] = []
|
| 389 |
+
remaining = dict(event_dict)
|
| 390 |
+
|
| 391 |
+
for key in self._key_order:
|
| 392 |
+
if key in remaining:
|
| 393 |
+
ordered.append((key, remaining.pop(key)))
|
| 394 |
+
elif not self._drop_missing:
|
| 395 |
+
ordered.append((key, None))
|
| 396 |
+
|
| 397 |
+
rest = sorted(remaining.items()) if self._sort_keys else list(remaining.items())
|
| 398 |
+
ordered.extend(rest)
|
| 399 |
+
|
| 400 |
+
parts = [
|
| 401 |
+
f"{k}={v!r}" if not isinstance(v, str) else f"{k}={v}" for k, v in ordered
|
| 402 |
+
]
|
| 403 |
+
return " ".join(parts)
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
class JSONRenderer:
|
| 407 |
+
"""Render the event dict as a JSON string.
|
| 408 |
+
|
| 409 |
+
Args:
|
| 410 |
+
serializer: JSON serialization function. Defaults to
|
| 411 |
+
:func:`json.dumps`.
|
| 412 |
+
**dumps_kw: Extra keyword arguments passed to *serializer*.
|
| 413 |
+
"""
|
| 414 |
+
|
| 415 |
+
__slots__ = ("_serializer", "_dumps_kw")
|
| 416 |
+
|
| 417 |
+
def __init__(
|
| 418 |
+
self,
|
| 419 |
+
serializer: Callable[..., str] = _json.dumps,
|
| 420 |
+
**dumps_kw: Any,
|
| 421 |
+
) -> None:
|
| 422 |
+
self._serializer = serializer
|
| 423 |
+
self._dumps_kw = dumps_kw
|
| 424 |
+
if "default" not in self._dumps_kw:
|
| 425 |
+
self._dumps_kw["default"] = _json_default
|
| 426 |
+
|
| 427 |
+
def __call__(self, logger: Any, method_name: str, event_dict: EventDict) -> str:
|
| 428 |
+
return self._serializer(event_dict, **self._dumps_kw)
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def _json_default(obj: Any) -> Any:
|
| 432 |
+
"""Fallback serializer for objects that ``json.dumps`` cannot handle."""
|
| 433 |
+
if isinstance(obj, datetime.datetime):
|
| 434 |
+
return obj.isoformat()
|
| 435 |
+
if isinstance(obj, datetime.date):
|
| 436 |
+
return obj.isoformat()
|
| 437 |
+
if isinstance(obj, set | frozenset):
|
| 438 |
+
return sorted(obj, key=str)
|
| 439 |
+
if isinstance(obj, bytes):
|
| 440 |
+
return obj.decode("utf-8", errors="replace")
|
| 441 |
+
return str(obj)
|
| 442 |
+
|
| 443 |
+
|
| 444 |
+
class ConsoleRenderer:
|
| 445 |
+
"""Render the event dict as colorized, loguru-style console output.
|
| 446 |
+
|
| 447 |
+
Output format::
|
| 448 |
+
|
| 449 |
+
2026-03-27 14:30:00.123 | INFO | event message key=val key=val
|
| 450 |
+
|
| 451 |
+
Args:
|
| 452 |
+
colors: Enable ANSI color codes. *None* auto-detects terminal
|
| 453 |
+
support.
|
| 454 |
+
pad_event: Pad the event field to this width for alignment.
|
| 455 |
+
level_styles: Override per-level ANSI color strings.
|
| 456 |
+
"""
|
| 457 |
+
|
| 458 |
+
__slots__ = ("_colors", "_pad_event", "_level_styles")
|
| 459 |
+
|
| 460 |
+
def __init__(
|
| 461 |
+
self,
|
| 462 |
+
colors: bool | None = None,
|
| 463 |
+
pad_event: int = 30,
|
| 464 |
+
level_styles: dict[str, str] | None = None,
|
| 465 |
+
) -> None:
|
| 466 |
+
self._colors = _supports_color() if colors is None else colors
|
| 467 |
+
self._pad_event = pad_event
|
| 468 |
+
self._level_styles = level_styles or _LEVEL_STYLES
|
| 469 |
+
|
| 470 |
+
def __call__(self, logger: Any, method_name: str, event_dict: EventDict) -> str:
|
| 471 |
+
# Extract well-known keys
|
| 472 |
+
event = str(event_dict.pop("event", ""))
|
| 473 |
+
level = event_dict.pop("level", method_name)
|
| 474 |
+
timestamp = event_dict.pop("timestamp", None)
|
| 475 |
+
event_dict.pop("logger", None)
|
| 476 |
+
exception = event_dict.pop("exception", None)
|
| 477 |
+
|
| 478 |
+
# Build timestamp string
|
| 479 |
+
if timestamp is None:
|
| 480 |
+
now = datetime.datetime.now()
|
| 481 |
+
ms = now.microsecond // 1000
|
| 482 |
+
ts_str = now.strftime("%Y-%m-%d %H:%M:%S.") + f"{ms:03d}"
|
| 483 |
+
elif isinstance(timestamp, str):
|
| 484 |
+
# Try to format ISO timestamps more compactly
|
| 485 |
+
ts_str = _compact_iso(timestamp)
|
| 486 |
+
elif isinstance(timestamp, float):
|
| 487 |
+
dt = datetime.datetime.fromtimestamp(timestamp)
|
| 488 |
+
ts_str = dt.strftime("%Y-%m-%d %H:%M:%S.") + f"{dt.microsecond // 1000:03d}"
|
| 489 |
+
else:
|
| 490 |
+
ts_str = str(timestamp)
|
| 491 |
+
|
| 492 |
+
# Build level label
|
| 493 |
+
level_str = _LEVEL_LABELS.get(level, f"{level.upper():<8s}")
|
| 494 |
+
|
| 495 |
+
# Build key=value pairs from remaining context
|
| 496 |
+
kv_parts: list[str] = []
|
| 497 |
+
for k, v in event_dict.items():
|
| 498 |
+
kv_parts.append(f"{k}={v!r}" if not isinstance(v, str) else f"{k}={v}")
|
| 499 |
+
kv_str = " ".join(kv_parts)
|
| 500 |
+
|
| 501 |
+
# Pad event
|
| 502 |
+
padded_event = event.ljust(self._pad_event) if kv_str else event
|
| 503 |
+
|
| 504 |
+
# Assemble
|
| 505 |
+
if self._colors:
|
| 506 |
+
msg_color = self._level_styles.get(level, _Colors.WHITE)
|
| 507 |
+
label_color = _LEVEL_LABEL_STYLES.get(level, _Colors.WHITE)
|
| 508 |
+
|
| 509 |
+
line = (
|
| 510 |
+
f"{_Colors.GREEN}{ts_str}{_Colors.RESET} | "
|
| 511 |
+
f"{label_color}{_Colors.BOLD}{level_str}{_Colors.RESET} | "
|
| 512 |
+
f"{msg_color}{padded_event}{_Colors.RESET}"
|
| 513 |
+
)
|
| 514 |
+
if kv_str:
|
| 515 |
+
line += f" {_Colors.DIM}{kv_str}{_Colors.RESET}"
|
| 516 |
+
if exception:
|
| 517 |
+
line += f"\n{_Colors.RED}{exception}{_Colors.RESET}"
|
| 518 |
+
else:
|
| 519 |
+
line = f"{ts_str} | {level_str} | {padded_event}"
|
| 520 |
+
if kv_str:
|
| 521 |
+
line += f" {kv_str}"
|
| 522 |
+
if exception:
|
| 523 |
+
line += f"\n{exception}"
|
| 524 |
+
|
| 525 |
+
return line
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def _compact_iso(ts: str) -> str:
|
| 529 |
+
"""Convert an ISO 8601 timestamp to ``YYYY-MM-DD HH:MM:SS.mmm`` format."""
|
| 530 |
+
try:
|
| 531 |
+
dt = datetime.datetime.fromisoformat(ts)
|
| 532 |
+
return dt.strftime("%Y-%m-%d %H:%M:%S.") + f"{dt.microsecond // 1000:03d}"
|
| 533 |
+
except (ValueError, AttributeError):
|
| 534 |
+
return ts
|
| 535 |
+
|
| 536 |
+
|
| 537 |
+
# ── BoundLogger ──────────────────────────────────────────────────────────────
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
class BoundLogger:
|
| 541 |
+
"""A logger that carries bound context through a processor pipeline.
|
| 542 |
+
|
| 543 |
+
Do not instantiate directly; use :func:`get_logger` or
|
| 544 |
+
:func:`wrap_logger` instead.
|
| 545 |
+
|
| 546 |
+
Args:
|
| 547 |
+
logger: The underlying logger instance (e.g. ``PrintLogger``).
|
| 548 |
+
processors: Ordered list of processors to run on each log event.
|
| 549 |
+
context: Initial context dictionary.
|
| 550 |
+
"""
|
| 551 |
+
|
| 552 |
+
__slots__ = ("_logger", "_processors", "_context")
|
| 553 |
+
|
| 554 |
+
def __init__(
|
| 555 |
+
self,
|
| 556 |
+
logger: Any,
|
| 557 |
+
processors: list[Processor],
|
| 558 |
+
context: dict[str, Any],
|
| 559 |
+
) -> None:
|
| 560 |
+
self._logger = logger
|
| 561 |
+
self._processors = processors
|
| 562 |
+
self._context = context
|
| 563 |
+
|
| 564 |
+
def bind(self, **new_values: Any) -> BoundLogger:
|
| 565 |
+
"""Return a new logger with *new_values* merged into the context."""
|
| 566 |
+
new_ctx = {**self._context, **new_values}
|
| 567 |
+
return BoundLogger(self._logger, self._processors, new_ctx)
|
| 568 |
+
|
| 569 |
+
def unbind(self, *keys: str) -> BoundLogger:
|
| 570 |
+
"""Return a new logger with *keys* removed from the context."""
|
| 571 |
+
new_ctx = {k: v for k, v in self._context.items() if k not in keys}
|
| 572 |
+
return BoundLogger(self._logger, self._processors, new_ctx)
|
| 573 |
+
|
| 574 |
+
def new(self, **new_values: Any) -> BoundLogger:
|
| 575 |
+
"""Return a new logger with *new_values* replacing the context."""
|
| 576 |
+
return BoundLogger(self._logger, self._processors, dict(new_values))
|
| 577 |
+
|
| 578 |
+
# ── Log methods ──
|
| 579 |
+
|
| 580 |
+
def debug(self, event: str | None = None, /, **kw: Any) -> None:
|
| 581 |
+
self._process("debug", event, kw)
|
| 582 |
+
|
| 583 |
+
def info(self, event: str | None = None, /, **kw: Any) -> None:
|
| 584 |
+
self._process("info", event, kw)
|
| 585 |
+
|
| 586 |
+
def warning(self, event: str | None = None, /, **kw: Any) -> None:
|
| 587 |
+
self._process("warning", event, kw)
|
| 588 |
+
|
| 589 |
+
def warn(self, event: str | None = None, /, **kw: Any) -> None:
|
| 590 |
+
self._process("warning", event, kw)
|
| 591 |
+
|
| 592 |
+
def error(self, event: str | None = None, /, **kw: Any) -> None:
|
| 593 |
+
self._process("error", event, kw)
|
| 594 |
+
|
| 595 |
+
def critical(self, event: str | None = None, /, **kw: Any) -> None:
|
| 596 |
+
self._process("critical", event, kw)
|
| 597 |
+
|
| 598 |
+
def fatal(self, event: str | None = None, /, **kw: Any) -> None:
|
| 599 |
+
self._process("critical", event, kw)
|
| 600 |
+
|
| 601 |
+
def exception(self, event: str | None = None, /, **kw: Any) -> None:
|
| 602 |
+
kw.setdefault("exc_info", True)
|
| 603 |
+
self._process("error", event, kw)
|
| 604 |
+
|
| 605 |
+
def log(self, level: int, event: str | None = None, /, **kw: Any) -> None:
|
| 606 |
+
method_name = _LEVEL_TO_NAME.get(level, "info")
|
| 607 |
+
self._process(method_name, event, kw)
|
| 608 |
+
|
| 609 |
+
# ── Internal ──
|
| 610 |
+
|
| 611 |
+
def _process(self, method_name: str, event: str | None, kw: dict[str, Any]) -> None:
|
| 612 |
+
"""Merge context, run processors, and emit via the underlying logger."""
|
| 613 |
+
event_dict: EventDict = {**self._context, **kw}
|
| 614 |
+
if event is not None:
|
| 615 |
+
event_dict["event"] = event
|
| 616 |
+
elif "event" not in event_dict:
|
| 617 |
+
event_dict["event"] = ""
|
| 618 |
+
|
| 619 |
+
try:
|
| 620 |
+
for proc in self._processors:
|
| 621 |
+
event_dict_or_str = proc(self._logger, method_name, event_dict)
|
| 622 |
+
if isinstance(event_dict_or_str, str):
|
| 623 |
+
# Final renderer returned a string -- emit and stop.
|
| 624 |
+
_emit(self._logger, method_name, event_dict_or_str)
|
| 625 |
+
return
|
| 626 |
+
event_dict = event_dict_or_str
|
| 627 |
+
except DropEvent:
|
| 628 |
+
return
|
| 629 |
+
|
| 630 |
+
# If no processor returned a string, emit the repr of event_dict.
|
| 631 |
+
_emit(self._logger, method_name, str(event_dict))
|
| 632 |
+
|
| 633 |
+
|
| 634 |
+
def _emit(logger: Any, method_name: str, message: str) -> None:
|
| 635 |
+
"""Dispatch *message* to the underlying logger."""
|
| 636 |
+
if isinstance(logger, logging.Logger):
|
| 637 |
+
level = _NAME_TO_LEVEL.get(method_name, logging.INFO)
|
| 638 |
+
logger.log(level, "%s", message)
|
| 639 |
+
else:
|
| 640 |
+
func = getattr(logger, method_name, None) or getattr(logger, "msg", None)
|
| 641 |
+
if func:
|
| 642 |
+
func(message)
|
| 643 |
+
|
| 644 |
+
|
| 645 |
+
# ── Configuration ────────────────────────────────────────────────────────────
|
| 646 |
+
|
| 647 |
+
|
| 648 |
+
@dataclasses.dataclass
|
| 649 |
+
class _Configuration:
|
| 650 |
+
"""Global structlog configuration."""
|
| 651 |
+
|
| 652 |
+
processors: list[Processor]
|
| 653 |
+
wrapper_class: type[BoundLogger]
|
| 654 |
+
context_class: type
|
| 655 |
+
logger_factory: LoggerFactory
|
| 656 |
+
cache_logger_on_first_use: bool
|
| 657 |
+
|
| 658 |
+
|
| 659 |
+
def _make_defaults() -> _Configuration:
|
| 660 |
+
return _Configuration(
|
| 661 |
+
processors=[add_log_level, TimeStamper(), ConsoleRenderer()],
|
| 662 |
+
wrapper_class=BoundLogger,
|
| 663 |
+
context_class=dict,
|
| 664 |
+
logger_factory=PrintLoggerFactory(),
|
| 665 |
+
cache_logger_on_first_use=True,
|
| 666 |
+
)
|
| 667 |
+
|
| 668 |
+
|
| 669 |
+
_config: _Configuration = _make_defaults()
|
| 670 |
+
_logger_cache: dict[Any, BoundLogger] = {}
|
| 671 |
+
|
| 672 |
+
|
| 673 |
+
def configure(
|
| 674 |
+
processors: list[Processor] | None = None,
|
| 675 |
+
wrapper_class: type[BoundLogger] | None = None,
|
| 676 |
+
context_class: type[dict] | None = None,
|
| 677 |
+
logger_factory: LoggerFactory | None = None,
|
| 678 |
+
cache_logger_on_first_use: bool | None = None,
|
| 679 |
+
) -> None:
|
| 680 |
+
"""Override the global configuration.
|
| 681 |
+
|
| 682 |
+
Only non-*None* arguments are changed. Call :func:`reset_defaults` to
|
| 683 |
+
restore factory settings.
|
| 684 |
+
|
| 685 |
+
Args:
|
| 686 |
+
processors: Ordered processor list.
|
| 687 |
+
wrapper_class: BoundLogger subclass to use.
|
| 688 |
+
context_class: Dict-like class for context storage.
|
| 689 |
+
logger_factory: Factory for the underlying logger.
|
| 690 |
+
cache_logger_on_first_use: Cache loggers returned by
|
| 691 |
+
:func:`get_logger`.
|
| 692 |
+
"""
|
| 693 |
+
global _config
|
| 694 |
+
_logger_cache.clear()
|
| 695 |
+
p = _config
|
| 696 |
+
_config = _Configuration(
|
| 697 |
+
processors=(processors if processors is not None else p.processors),
|
| 698 |
+
wrapper_class=(wrapper_class if wrapper_class is not None else p.wrapper_class),
|
| 699 |
+
context_class=(context_class if context_class is not None else p.context_class),
|
| 700 |
+
logger_factory=(
|
| 701 |
+
logger_factory if logger_factory is not None else p.logger_factory
|
| 702 |
+
),
|
| 703 |
+
cache_logger_on_first_use=(
|
| 704 |
+
cache_logger_on_first_use
|
| 705 |
+
if cache_logger_on_first_use is not None
|
| 706 |
+
else p.cache_logger_on_first_use
|
| 707 |
+
),
|
| 708 |
+
)
|
| 709 |
+
|
| 710 |
+
|
| 711 |
+
def reset_defaults() -> None:
|
| 712 |
+
"""Restore the global configuration to factory defaults."""
|
| 713 |
+
global _config
|
| 714 |
+
_logger_cache.clear()
|
| 715 |
+
_config = _make_defaults()
|
| 716 |
+
|
| 717 |
+
|
| 718 |
+
def get_config() -> _Configuration:
|
| 719 |
+
"""Return the current global configuration (read-only snapshot)."""
|
| 720 |
+
return _config
|
| 721 |
+
|
| 722 |
+
|
| 723 |
+
# ── Logger Creation ──────────────────────────────────────────────────────────
|
| 724 |
+
|
| 725 |
+
|
| 726 |
+
def get_logger(*args: Any, **initial_values: Any) -> BoundLogger:
|
| 727 |
+
"""Create a :class:`BoundLogger` using the global configuration.
|
| 728 |
+
|
| 729 |
+
Positional arguments are forwarded to the logger factory (e.g. a
|
| 730 |
+
logger name). Keyword arguments become the initial bound context.
|
| 731 |
+
|
| 732 |
+
Returns:
|
| 733 |
+
A configured :class:`BoundLogger`.
|
| 734 |
+
"""
|
| 735 |
+
cache_key = args if not initial_values else None
|
| 736 |
+
|
| 737 |
+
if cache_key is not None and _config.cache_logger_on_first_use:
|
| 738 |
+
cached = _logger_cache.get(cache_key)
|
| 739 |
+
if cached is not None:
|
| 740 |
+
return cached
|
| 741 |
+
|
| 742 |
+
underlying = _config.logger_factory(*args)
|
| 743 |
+
ctx = (
|
| 744 |
+
_config.context_class(initial_values)
|
| 745 |
+
if initial_values
|
| 746 |
+
else _config.context_class()
|
| 747 |
+
)
|
| 748 |
+
bound = _config.wrapper_class(
|
| 749 |
+
logger=underlying,
|
| 750 |
+
processors=list(_config.processors),
|
| 751 |
+
context=ctx,
|
| 752 |
+
)
|
| 753 |
+
|
| 754 |
+
if cache_key is not None and _config.cache_logger_on_first_use:
|
| 755 |
+
_logger_cache[cache_key] = bound
|
| 756 |
+
|
| 757 |
+
return bound
|
| 758 |
+
|
| 759 |
+
|
| 760 |
+
def wrap_logger(
|
| 761 |
+
logger: Any,
|
| 762 |
+
processors: list[Processor] | None = None,
|
| 763 |
+
**initial_values: Any,
|
| 764 |
+
) -> BoundLogger:
|
| 765 |
+
"""Wrap an existing logger with a processor pipeline.
|
| 766 |
+
|
| 767 |
+
Args:
|
| 768 |
+
logger: Any object with ``debug``/``info``/... methods.
|
| 769 |
+
processors: Processor list. Defaults to the global config.
|
| 770 |
+
**initial_values: Initial bound context.
|
| 771 |
+
|
| 772 |
+
Returns:
|
| 773 |
+
A :class:`BoundLogger` wrapping *logger*.
|
| 774 |
+
"""
|
| 775 |
+
procs = processors if processors is not None else list(_config.processors)
|
| 776 |
+
ctx = dict(initial_values)
|
| 777 |
+
return BoundLogger(logger=logger, processors=procs, context=ctx)
|
| 778 |
+
|
| 779 |
+
|
| 780 |
+
# ── Convenience Setup ────────────────────────────────────────────────────────
|
| 781 |
+
|
| 782 |
+
|
| 783 |
+
def _resolve_level(level: int | str) -> int:
|
| 784 |
+
"""Convert a level name or int to a logging level int."""
|
| 785 |
+
if isinstance(level, int):
|
| 786 |
+
return level
|
| 787 |
+
return _NAME_TO_LEVEL.get(level.lower(), logging.INFO)
|
| 788 |
+
|
| 789 |
+
|
| 790 |
+
def setup_logging(
|
| 791 |
+
level: int | str = logging.INFO,
|
| 792 |
+
renderer: str = "console",
|
| 793 |
+
colors: bool | None = None,
|
| 794 |
+
processors: list[Processor] | None = None,
|
| 795 |
+
logger_name: str | None = None,
|
| 796 |
+
stream: IO[str] | None = None,
|
| 797 |
+
) -> BoundLogger:
|
| 798 |
+
"""One-call logging setup with stdlib integration.
|
| 799 |
+
|
| 800 |
+
Configures both stdlib ``logging`` and the structlog processor
|
| 801 |
+
pipeline in a single call.
|
| 802 |
+
|
| 803 |
+
Args:
|
| 804 |
+
level: Log level (name or int). Defaults to ``INFO``.
|
| 805 |
+
renderer: Output renderer: ``"console"``, ``"json"``, or ``"kv"``.
|
| 806 |
+
colors: Enable ANSI colors. *None* auto-detects.
|
| 807 |
+
processors: Custom processor list. Overrides *renderer* if given.
|
| 808 |
+
logger_name: stdlib logger name.
|
| 809 |
+
stream: Output stream. Defaults to ``sys.stderr``.
|
| 810 |
+
|
| 811 |
+
Returns:
|
| 812 |
+
A ready-to-use :class:`BoundLogger`.
|
| 813 |
+
"""
|
| 814 |
+
resolved_level = _resolve_level(level)
|
| 815 |
+
out = stream or sys.stderr
|
| 816 |
+
|
| 817 |
+
if processors is None:
|
| 818 |
+
base: list[Processor] = [add_log_level, TimeStamper()]
|
| 819 |
+
if renderer == "json":
|
| 820 |
+
base.append(JSONRenderer())
|
| 821 |
+
elif renderer == "kv":
|
| 822 |
+
base.append(KeyValueRenderer(key_order=["event", "level", "timestamp"]))
|
| 823 |
+
else:
|
| 824 |
+
base.append(ConsoleRenderer(colors=colors))
|
| 825 |
+
processors = base
|
| 826 |
+
|
| 827 |
+
# Configure stdlib logging to pass through rendered strings.
|
| 828 |
+
stdlib_logger = logging.getLogger(logger_name)
|
| 829 |
+
stdlib_logger.setLevel(resolved_level)
|
| 830 |
+
stdlib_logger.propagate = False
|
| 831 |
+
|
| 832 |
+
# Remove existing handlers to avoid duplicate output on repeated calls.
|
| 833 |
+
stdlib_logger.handlers.clear()
|
| 834 |
+
|
| 835 |
+
handler = logging.StreamHandler(out)
|
| 836 |
+
handler.setLevel(resolved_level)
|
| 837 |
+
handler.setFormatter(logging.Formatter("%(message)s"))
|
| 838 |
+
stdlib_logger.addHandler(handler)
|
| 839 |
+
|
| 840 |
+
configure(
|
| 841 |
+
processors=processors,
|
| 842 |
+
logger_factory=StdlibLoggerFactory(name=logger_name),
|
| 843 |
+
)
|
| 844 |
+
|
| 845 |
+
return get_logger()
|
| 846 |
+
|
| 847 |
+
|
| 848 |
+
# ── Utilities ────────────────────────────────────────────────────────────────
|
| 849 |
+
|
| 850 |
+
|
| 851 |
+
def truncate_string(s: str, max_length: int, suffix: str = "...") -> str:
|
| 852 |
+
"""Truncate *s* to *max_length*, appending a count of remaining chars.
|
| 853 |
+
|
| 854 |
+
Args:
|
| 855 |
+
s: The string to truncate.
|
| 856 |
+
max_length: Maximum number of characters to keep.
|
| 857 |
+
suffix: Separator between the kept text and the count.
|
| 858 |
+
|
| 859 |
+
Returns:
|
| 860 |
+
The original string if short enough, otherwise truncated with a
|
| 861 |
+
``"...[N more chars]"`` suffix.
|
| 862 |
+
"""
|
| 863 |
+
if len(s) <= max_length:
|
| 864 |
+
return s
|
| 865 |
+
remaining = len(s) - max_length
|
| 866 |
+
return f"{s[:max_length]}{suffix}[{remaining} more chars]"
|
| 867 |
+
|
| 868 |
+
|
| 869 |
+
def truncate_base64(data_url: str, max_length: int = 100) -> str:
|
| 870 |
+
"""Truncate base64 data-URLs for cleaner logging.
|
| 871 |
+
|
| 872 |
+
Args:
|
| 873 |
+
data_url: A ``data:`` URL or any string.
|
| 874 |
+
max_length: Maximum base64 payload chars to keep.
|
| 875 |
+
|
| 876 |
+
Returns:
|
| 877 |
+
The truncated URL, or the original string if not a data-URL.
|
| 878 |
+
"""
|
| 879 |
+
if not data_url.startswith("data:"):
|
| 880 |
+
return data_url
|
| 881 |
+
if ";base64," in data_url:
|
| 882 |
+
header, base64_data = data_url.split(";base64,", 1)
|
| 883 |
+
if len(base64_data) > max_length:
|
| 884 |
+
remaining = len(base64_data) - max_length
|
| 885 |
+
return (
|
| 886 |
+
f"{header};base64,{base64_data[:max_length]}...[{remaining} more chars]"
|
| 887 |
+
)
|
| 888 |
+
return data_url
|
|
@@ -0,0 +1,475 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.1.0"
|
| 3 |
+
# deps = []
|
| 4 |
+
# tier = "simple"
|
| 5 |
+
# category = "network"
|
| 6 |
+
# note = "Install/update via `zerodep add useragent`"
|
| 7 |
+
# ///
|
| 8 |
+
|
| 9 |
+
"""Lightweight User-Agent generator for Chrome/Edge with Client Hints.
|
| 10 |
+
|
| 11 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 12 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 13 |
+
|
| 14 |
+
Generates realistic browser UA strings and matching ``Sec-CH-UA-*`` headers.
|
| 15 |
+
Covers Windows, macOS, Linux (desktop) and Android (mobile) platforms with
|
| 16 |
+
Chrome and Edge browsers only -- this is intentionally minimal.
|
| 17 |
+
|
| 18 |
+
Inspired by `ua-generator <https://github.com/iamdual/ua-generator>`_ (Apache-2.0).
|
| 19 |
+
|
| 20 |
+
Zero external dependencies -- stdlib ``random`` only.
|
| 21 |
+
|
| 22 |
+
Basic usage::
|
| 23 |
+
|
| 24 |
+
from useragent import generate
|
| 25 |
+
|
| 26 |
+
ua = generate(browser="chrome", device="desktop")
|
| 27 |
+
print(ua.text) # full User-Agent string
|
| 28 |
+
print(ua.headers.get()) # dict with UA + Client Hints headers
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
from __future__ import annotations
|
| 32 |
+
|
| 33 |
+
import random
|
| 34 |
+
from typing import Sequence, Union
|
| 35 |
+
|
| 36 |
+
__all__ = ["UserAgent", "generate"]
|
| 37 |
+
|
| 38 |
+
__version__ = "0.1.0"
|
| 39 |
+
|
| 40 |
+
# ---------------------------------------------------------------------------
|
| 41 |
+
# Version data
|
| 42 |
+
# ---------------------------------------------------------------------------
|
| 43 |
+
|
| 44 |
+
# (major, build) pairs. Patch is randomised at generation time.
|
| 45 |
+
# Sources:
|
| 46 |
+
# Chrome -- https://chromereleases.googleblog.com/search/label/Stable%20updates
|
| 47 |
+
# Edge -- https://learn.microsoft.com/en-us/deployedge/microsoft-edge-release-schedule
|
| 48 |
+
|
| 49 |
+
_CHROME_VERSIONS: list[tuple[int, int]] = [
|
| 50 |
+
(120, 6099),
|
| 51 |
+
(121, 6167),
|
| 52 |
+
(122, 6261),
|
| 53 |
+
(123, 6312),
|
| 54 |
+
(124, 6367),
|
| 55 |
+
(125, 6422),
|
| 56 |
+
(126, 6478),
|
| 57 |
+
(127, 6533),
|
| 58 |
+
(128, 6613),
|
| 59 |
+
(129, 6668),
|
| 60 |
+
(130, 6723),
|
| 61 |
+
(131, 6778),
|
| 62 |
+
(132, 6834),
|
| 63 |
+
(133, 6943),
|
| 64 |
+
(134, 6998),
|
| 65 |
+
(135, 7049),
|
| 66 |
+
(136, 7103),
|
| 67 |
+
(137, 7151),
|
| 68 |
+
(138, 7204),
|
| 69 |
+
(139, 7258),
|
| 70 |
+
(140, 7339),
|
| 71 |
+
(141, 7390),
|
| 72 |
+
(142, 7444),
|
| 73 |
+
(143, 7499),
|
| 74 |
+
(144, 7559),
|
| 75 |
+
(145, 7632),
|
| 76 |
+
(146, 7680),
|
| 77 |
+
]
|
| 78 |
+
|
| 79 |
+
_EDGE_VERSIONS: list[tuple[int, int]] = [
|
| 80 |
+
(120, 2210),
|
| 81 |
+
(121, 2277),
|
| 82 |
+
(122, 2365),
|
| 83 |
+
(123, 2420),
|
| 84 |
+
(124, 2478),
|
| 85 |
+
(125, 2535),
|
| 86 |
+
(126, 2592),
|
| 87 |
+
(127, 2651),
|
| 88 |
+
(128, 2739),
|
| 89 |
+
(129, 2792),
|
| 90 |
+
(130, 2849),
|
| 91 |
+
(131, 2903),
|
| 92 |
+
(132, 2957),
|
| 93 |
+
(133, 3065),
|
| 94 |
+
(134, 3124),
|
| 95 |
+
(135, 3179),
|
| 96 |
+
(136, 3240),
|
| 97 |
+
(137, 3296),
|
| 98 |
+
(138, 3351),
|
| 99 |
+
(139, 3405),
|
| 100 |
+
(140, 3485),
|
| 101 |
+
(141, 3537),
|
| 102 |
+
(142, 3595),
|
| 103 |
+
(143, 3650),
|
| 104 |
+
(144, 3719),
|
| 105 |
+
(145, 3800),
|
| 106 |
+
(146, 3856),
|
| 107 |
+
]
|
| 108 |
+
|
| 109 |
+
# Windows NT versions: (nt_major, nt_minor, ch_platform_range)
|
| 110 |
+
_WINDOWS_VERSIONS: list[tuple[float, tuple[int, int]]] = [
|
| 111 |
+
(10.0, (1, 10)), # Windows 10
|
| 112 |
+
(10.0, (13, 16)), # Windows 11
|
| 113 |
+
]
|
| 114 |
+
|
| 115 |
+
# macOS versions: (major, minor, max_build)
|
| 116 |
+
_MACOS_VERSIONS: list[tuple[int, int, int]] = [
|
| 117 |
+
(13, 6, 9),
|
| 118 |
+
(13, 7, 8),
|
| 119 |
+
(14, 4, 1),
|
| 120 |
+
(14, 5, 0),
|
| 121 |
+
(14, 6, 1),
|
| 122 |
+
(14, 7, 8),
|
| 123 |
+
(14, 8, 4),
|
| 124 |
+
(15, 0, 1),
|
| 125 |
+
(15, 1, 1),
|
| 126 |
+
(15, 2, 1),
|
| 127 |
+
(15, 3, 1),
|
| 128 |
+
(15, 4, 1),
|
| 129 |
+
(15, 5, 0),
|
| 130 |
+
(15, 6, 1),
|
| 131 |
+
(15, 7, 4),
|
| 132 |
+
(26, 0, 1),
|
| 133 |
+
(26, 1, 0),
|
| 134 |
+
(26, 2, 0),
|
| 135 |
+
(26, 3, 2),
|
| 136 |
+
]
|
| 137 |
+
|
| 138 |
+
# Android model strings (Samsung subset -- most common vendor)
|
| 139 |
+
_ANDROID_MODELS: tuple[str, ...] = (
|
| 140 |
+
"SM-G991B",
|
| 141 |
+
"SM-G996B",
|
| 142 |
+
"SM-G998B",
|
| 143 |
+
"SM-A526B",
|
| 144 |
+
"SM-A536B",
|
| 145 |
+
"SM-S901B",
|
| 146 |
+
"SM-S906B",
|
| 147 |
+
"SM-S908B",
|
| 148 |
+
"SM-S911B",
|
| 149 |
+
"SM-S916B",
|
| 150 |
+
"SM-S918B",
|
| 151 |
+
"SM-S921B",
|
| 152 |
+
"SM-S926B",
|
| 153 |
+
"SM-S928B",
|
| 154 |
+
"SM-A546B",
|
| 155 |
+
"SM-A556B",
|
| 156 |
+
"SM-A346B",
|
| 157 |
+
"SM-A256B",
|
| 158 |
+
"SM-A156B",
|
| 159 |
+
"SM-G781B",
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
_DESKTOP_PLATFORMS = ("windows", "macos", "linux")
|
| 163 |
+
_MOBILE_PLATFORMS = ("android",)
|
| 164 |
+
_ALL_PLATFORMS = _DESKTOP_PLATFORMS + _MOBILE_PLATFORMS
|
| 165 |
+
|
| 166 |
+
_WEBKIT = "537.36"
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
# ---------------------------------------------------------------------------
|
| 170 |
+
# Internal helpers
|
| 171 |
+
# ---------------------------------------------------------------------------
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def _pick_chrome_version() -> tuple[int, int, int, int]:
|
| 175 |
+
"""Return (major, minor=0, build, patch)."""
|
| 176 |
+
major, build = random.choice(_CHROME_VERSIONS)
|
| 177 |
+
return (major, 0, build, random.randint(0, 255))
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def _pick_edge_version() -> tuple[int, int, int, int]:
|
| 181 |
+
major, build = random.choice(_EDGE_VERSIONS)
|
| 182 |
+
return (major, 0, build, random.randint(0, 99))
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def _fmt_ver(v: tuple[int, ...], n: int = 4, sep: str = ".") -> str:
|
| 186 |
+
"""Format version tuple to string, padding with 0s up to *n* parts."""
|
| 187 |
+
parts = list(v[:n])
|
| 188 |
+
while len(parts) < n:
|
| 189 |
+
parts.append(0)
|
| 190 |
+
return sep.join(str(p) for p in parts)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _pick_platform(device: str | None) -> str:
|
| 194 |
+
if device == "desktop":
|
| 195 |
+
return random.choice(_DESKTOP_PLATFORMS)
|
| 196 |
+
if device == "mobile":
|
| 197 |
+
return random.choice(_MOBILE_PLATFORMS)
|
| 198 |
+
return random.choice(_ALL_PLATFORMS)
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def _build_ua_string(
|
| 202 |
+
browser: str,
|
| 203 |
+
platform: str,
|
| 204 |
+
ver: tuple[int, int, int, int],
|
| 205 |
+
) -> str:
|
| 206 |
+
"""Build a User-Agent string from browser/platform/version."""
|
| 207 |
+
chrome_str = _fmt_ver(ver)
|
| 208 |
+
|
| 209 |
+
if platform == "windows":
|
| 210 |
+
nt = random.choice(_WINDOWS_VERSIONS)
|
| 211 |
+
nt_str = f"{nt[0]:.1f}".replace(".0", ".0") # "10.0"
|
| 212 |
+
base = (
|
| 213 |
+
f"Mozilla/5.0 (Windows NT {nt_str}; Win64; x64) "
|
| 214 |
+
f"AppleWebKit/{_WEBKIT} (KHTML, like Gecko) "
|
| 215 |
+
f"Chrome/{chrome_str} Safari/{_WEBKIT}"
|
| 216 |
+
)
|
| 217 |
+
if browser == "edge":
|
| 218 |
+
base += f" Edg/{chrome_str}"
|
| 219 |
+
return base
|
| 220 |
+
|
| 221 |
+
if platform == "linux":
|
| 222 |
+
tpl = random.choice(
|
| 223 |
+
(
|
| 224 |
+
f"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/{_WEBKIT} "
|
| 225 |
+
f"(KHTML, like Gecko) Chrome/{chrome_str} Safari/{_WEBKIT}",
|
| 226 |
+
f"Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/{_WEBKIT} "
|
| 227 |
+
f"(KHTML, like Gecko) Chrome/{chrome_str} Safari/{_WEBKIT}",
|
| 228 |
+
)
|
| 229 |
+
)
|
| 230 |
+
if browser == "edge":
|
| 231 |
+
tpl += f" Edg/{chrome_str}"
|
| 232 |
+
return tpl
|
| 233 |
+
|
| 234 |
+
if platform == "macos":
|
| 235 |
+
mv = random.choice(_MACOS_VERSIONS)
|
| 236 |
+
build = random.randint(0, mv[2]) if mv[2] > 0 else 0
|
| 237 |
+
mac_str = f"{mv[0]}_{mv[1]}"
|
| 238 |
+
if build:
|
| 239 |
+
mac_str += f"_{build}"
|
| 240 |
+
base = (
|
| 241 |
+
f"Mozilla/5.0 (Macintosh; Intel Mac OS X {mac_str}) "
|
| 242 |
+
f"AppleWebKit/{_WEBKIT} (KHTML, like Gecko) "
|
| 243 |
+
f"Chrome/{chrome_str} Safari/{_WEBKIT}"
|
| 244 |
+
)
|
| 245 |
+
if browser == "edge":
|
| 246 |
+
base += f" Edg/{chrome_str}"
|
| 247 |
+
return base
|
| 248 |
+
|
| 249 |
+
if platform == "android":
|
| 250 |
+
android_ver = random.randint(12, 16)
|
| 251 |
+
model = random.choice(_ANDROID_MODELS)
|
| 252 |
+
base = (
|
| 253 |
+
f"Mozilla/5.0 (Linux; Android {android_ver}; {model}) "
|
| 254 |
+
f"AppleWebKit/{_WEBKIT} (KHTML, like Gecko) "
|
| 255 |
+
f"Chrome/{chrome_str} Mobile Safari/{_WEBKIT}"
|
| 256 |
+
)
|
| 257 |
+
if browser == "edge":
|
| 258 |
+
base += f" EdgA/{chrome_str}"
|
| 259 |
+
return base
|
| 260 |
+
|
| 261 |
+
raise ValueError(f"Unsupported platform: {platform}")
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
# ---------------------------------------------------------------------------
|
| 265 |
+
# Client Hints serialization
|
| 266 |
+
# ---------------------------------------------------------------------------
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def _ch_bool(val: bool) -> str:
|
| 270 |
+
return "?1" if val else "?0"
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _ch_string(val: str) -> str:
|
| 274 |
+
return f'"{val}"'
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def _ch_brand_list(brands: list[dict[str, str]]) -> str:
|
| 278 |
+
return ", ".join(f'"{b["brand"]}";v="{b["version"]}"' for b in brands)
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
# ---------------------------------------------------------------------------
|
| 282 |
+
# Public API
|
| 283 |
+
# ---------------------------------------------------------------------------
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
class UserAgent:
|
| 287 |
+
"""A generated user-agent with matching Client Hints headers.
|
| 288 |
+
|
| 289 |
+
Attributes:
|
| 290 |
+
browser: Browser name (``"chrome"`` or ``"edge"``).
|
| 291 |
+
platform: Platform name (``"windows"``, ``"macos"``, ``"linux"``,
|
| 292 |
+
or ``"android"``).
|
| 293 |
+
version: 4-tuple ``(major, minor, build, patch)``.
|
| 294 |
+
text: The full User-Agent string.
|
| 295 |
+
headers: A :class:`_Headers` instance for Client Hints.
|
| 296 |
+
|
| 297 |
+
Example::
|
| 298 |
+
|
| 299 |
+
ua = generate(browser=["chrome", "edge"])
|
| 300 |
+
ua.headers.accept_ch("Sec-CH-UA-Platform-Version")
|
| 301 |
+
headers = ua.headers.get()
|
| 302 |
+
"""
|
| 303 |
+
|
| 304 |
+
def __init__(
|
| 305 |
+
self,
|
| 306 |
+
*,
|
| 307 |
+
browser: str,
|
| 308 |
+
platform: str,
|
| 309 |
+
version: tuple[int, int, int, int],
|
| 310 |
+
ua_string: str,
|
| 311 |
+
) -> None:
|
| 312 |
+
self.browser = browser
|
| 313 |
+
self.platform = platform
|
| 314 |
+
self.version = version
|
| 315 |
+
self.text = ua_string
|
| 316 |
+
self.headers = _Headers(self)
|
| 317 |
+
|
| 318 |
+
# -- Client Hints data methods --
|
| 319 |
+
|
| 320 |
+
def _is_mobile(self) -> bool:
|
| 321 |
+
return self.platform in _MOBILE_PLATFORMS
|
| 322 |
+
|
| 323 |
+
def _ch_platform_name(self) -> str:
|
| 324 |
+
mapping = {
|
| 325 |
+
"windows": "Windows",
|
| 326 |
+
"macos": "macOS",
|
| 327 |
+
"linux": "Linux",
|
| 328 |
+
"android": "Android",
|
| 329 |
+
}
|
| 330 |
+
return mapping.get(self.platform, self.platform.title())
|
| 331 |
+
|
| 332 |
+
def _ch_platform_version(self) -> str:
|
| 333 |
+
if self.platform == "windows":
|
| 334 |
+
lo, hi = random.choice(_WINDOWS_VERSIONS)[1]
|
| 335 |
+
return f"{random.randint(lo, hi)}.0.0"
|
| 336 |
+
if self.platform == "macos":
|
| 337 |
+
mv = random.choice(_MACOS_VERSIONS)
|
| 338 |
+
return f"{mv[0]}.{mv[1]}.{random.randint(0, max(mv[2], 1))}"
|
| 339 |
+
if self.platform == "android":
|
| 340 |
+
return f"{random.randint(12, 16)}.0.0"
|
| 341 |
+
# Linux
|
| 342 |
+
return ""
|
| 343 |
+
|
| 344 |
+
def _ch_brands(self, full_version: bool = False) -> list[dict[str, str]]:
|
| 345 |
+
ver_str = _fmt_ver(self.version) if full_version else str(self.version[0])
|
| 346 |
+
filler_ver = "99.0.0.0" if full_version else "99"
|
| 347 |
+
brands: list[dict[str, str]] = [{"brand": "Not A(Brand", "version": filler_ver}]
|
| 348 |
+
if self.browser == "chrome":
|
| 349 |
+
brands.append({"brand": "Chromium", "version": ver_str})
|
| 350 |
+
brands.append({"brand": "Google Chrome", "version": ver_str})
|
| 351 |
+
elif self.browser == "edge":
|
| 352 |
+
brands.append({"brand": "Chromium", "version": ver_str})
|
| 353 |
+
brands.append({"brand": "Microsoft Edge", "version": ver_str})
|
| 354 |
+
return brands
|
| 355 |
+
|
| 356 |
+
def _ch_architecture(self) -> str:
|
| 357 |
+
if self.platform in ("android",):
|
| 358 |
+
return "arm"
|
| 359 |
+
if self.platform == "macos":
|
| 360 |
+
return random.choice(("arm", "x86", "arm", "arm"))
|
| 361 |
+
return "x86"
|
| 362 |
+
|
| 363 |
+
def _ch_bitness(self) -> str:
|
| 364 |
+
if self.platform == "android":
|
| 365 |
+
return random.choice(("32", "64", "32", "32"))
|
| 366 |
+
return "64"
|
| 367 |
+
|
| 368 |
+
def _ch_model(self) -> str:
|
| 369 |
+
if self.platform == "android":
|
| 370 |
+
return random.choice(_ANDROID_MODELS)
|
| 371 |
+
return ""
|
| 372 |
+
|
| 373 |
+
def __str__(self) -> str:
|
| 374 |
+
return self.text
|
| 375 |
+
|
| 376 |
+
def __repr__(self) -> str:
|
| 377 |
+
return f"UserAgent(browser={self.browser!r}, platform={self.platform!r})"
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
class _Headers:
|
| 381 |
+
"""Lazy header builder matching the ``ua_generator.Headers`` interface."""
|
| 382 |
+
|
| 383 |
+
def __init__(self, ua: UserAgent) -> None:
|
| 384 |
+
self._ua = ua
|
| 385 |
+
self._headers: dict[str, str] = {}
|
| 386 |
+
self._initialised = False
|
| 387 |
+
|
| 388 |
+
def _reset(self) -> None:
|
| 389 |
+
self._initialised = True
|
| 390 |
+
self._headers = {"user-agent": self._ua.text}
|
| 391 |
+
# Low-entropy hints (always included for Chromium)
|
| 392 |
+
self._add("sec-ch-ua")
|
| 393 |
+
self._add("sec-ch-ua-mobile")
|
| 394 |
+
self._add("sec-ch-ua-platform")
|
| 395 |
+
|
| 396 |
+
def _add(self, key: str) -> None:
|
| 397 |
+
ua = self._ua
|
| 398 |
+
if key == "sec-ch-ua":
|
| 399 |
+
self._headers[key] = _ch_brand_list(ua._ch_brands(full_version=False))
|
| 400 |
+
elif key == "sec-ch-ua-full-version-list":
|
| 401 |
+
self._headers[key] = _ch_brand_list(ua._ch_brands(full_version=True))
|
| 402 |
+
elif key == "sec-ch-ua-platform":
|
| 403 |
+
self._headers[key] = _ch_string(ua._ch_platform_name())
|
| 404 |
+
elif key == "sec-ch-ua-platform-version":
|
| 405 |
+
self._headers[key] = _ch_string(ua._ch_platform_version())
|
| 406 |
+
elif key == "sec-ch-ua-mobile":
|
| 407 |
+
self._headers[key] = _ch_bool(ua._is_mobile())
|
| 408 |
+
elif key == "sec-ch-ua-arch":
|
| 409 |
+
self._headers[key] = _ch_string(ua._ch_architecture())
|
| 410 |
+
elif key == "sec-ch-ua-bitness":
|
| 411 |
+
self._headers[key] = _ch_string(ua._ch_bitness())
|
| 412 |
+
elif key == "sec-ch-ua-model":
|
| 413 |
+
self._headers[key] = _ch_string(ua._ch_model())
|
| 414 |
+
|
| 415 |
+
def accept_ch(self, val: str) -> None:
|
| 416 |
+
"""Process an ``Accept-CH`` header value and populate matching hints.
|
| 417 |
+
|
| 418 |
+
Args:
|
| 419 |
+
val: Comma-separated list of hint names
|
| 420 |
+
(e.g. ``"Sec-CH-UA-Platform-Version, Sec-CH-UA-Arch"``).
|
| 421 |
+
"""
|
| 422 |
+
self._reset()
|
| 423 |
+
for hint in val.split(","):
|
| 424 |
+
self._add(hint.strip().lower())
|
| 425 |
+
|
| 426 |
+
def get(self) -> dict[str, str]:
|
| 427 |
+
"""Return all generated headers as a dict."""
|
| 428 |
+
if not self._initialised:
|
| 429 |
+
self._reset()
|
| 430 |
+
return self._headers
|
| 431 |
+
|
| 432 |
+
|
| 433 |
+
def generate(
|
| 434 |
+
*,
|
| 435 |
+
browser: Union[str, Sequence[str]] | None = None,
|
| 436 |
+
device: str | None = None,
|
| 437 |
+
) -> UserAgent:
|
| 438 |
+
"""Generate a random User-Agent with matching Client Hints.
|
| 439 |
+
|
| 440 |
+
Args:
|
| 441 |
+
browser: Browser name or list of names to pick from.
|
| 442 |
+
Supported: ``"chrome"``, ``"edge"``. Defaults to both.
|
| 443 |
+
device: Device type -- ``"desktop"`` or ``"mobile"``.
|
| 444 |
+
Defaults to random selection across all platforms.
|
| 445 |
+
|
| 446 |
+
Returns:
|
| 447 |
+
A :class:`UserAgent` instance.
|
| 448 |
+
|
| 449 |
+
Raises:
|
| 450 |
+
ValueError: If *browser* is not ``"chrome"`` or ``"edge"``.
|
| 451 |
+
"""
|
| 452 |
+
# Resolve browser
|
| 453 |
+
if browser is None:
|
| 454 |
+
chosen_browser = random.choice(("chrome", "edge"))
|
| 455 |
+
elif isinstance(browser, str):
|
| 456 |
+
chosen_browser = browser
|
| 457 |
+
else:
|
| 458 |
+
chosen_browser = random.choice(list(browser))
|
| 459 |
+
|
| 460 |
+
if chosen_browser not in ("chrome", "edge"):
|
| 461 |
+
raise ValueError(f"Unsupported browser: {chosen_browser!r}")
|
| 462 |
+
|
| 463 |
+
# Resolve platform
|
| 464 |
+
platform = _pick_platform(device)
|
| 465 |
+
|
| 466 |
+
# Pick version
|
| 467 |
+
if chosen_browser == "chrome":
|
| 468 |
+
ver = _pick_chrome_version()
|
| 469 |
+
else:
|
| 470 |
+
ver = _pick_edge_version()
|
| 471 |
+
|
| 472 |
+
ua_string = _build_ua_string(chosen_browser, platform, ver)
|
| 473 |
+
return UserAgent(
|
| 474 |
+
browser=chosen_browser, platform=platform, version=ver, ua_string=ua_string
|
| 475 |
+
)
|
|
@@ -0,0 +1,1124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// zerodep
|
| 2 |
+
# version = "0.3.1"
|
| 3 |
+
# deps = []
|
| 4 |
+
# tier = "subsystem"
|
| 5 |
+
# category = "serialization"
|
| 6 |
+
# note = "Install/update via `zerodep add yaml`"
|
| 7 |
+
# ///
|
| 8 |
+
|
| 9 |
+
"""YAML parser and serializer (common subset) — zero dependencies, stdlib only, Python 3.10+.
|
| 10 |
+
|
| 11 |
+
Part of zerodep: https://github.com/Oaklight/zerodep
|
| 12 |
+
Copyright (c) 2026 Peng Ding. MIT License.
|
| 13 |
+
|
| 14 |
+
Supports the most commonly used YAML features: mappings, sequences,
|
| 15 |
+
scalars (str/int/float/bool/null), flow style, block scalars,
|
| 16 |
+
multi-document streams, and comments.
|
| 17 |
+
|
| 18 |
+
Does NOT implement: anchors/aliases, tags, merge keys, complex keys.
|
| 19 |
+
|
| 20 |
+
Example::
|
| 21 |
+
|
| 22 |
+
data = load("name: Alice\nage: 30")
|
| 23 |
+
# {'name': 'Alice', 'age': 30}
|
| 24 |
+
print(dump(data))
|
| 25 |
+
# age: 30
|
| 26 |
+
# name: Alice
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
from __future__ import annotations
|
| 30 |
+
|
| 31 |
+
import math
|
| 32 |
+
import re
|
| 33 |
+
from typing import IO, Any, Iterator, overload
|
| 34 |
+
|
| 35 |
+
__all__ = [
|
| 36 |
+
"YAMLError",
|
| 37 |
+
"load",
|
| 38 |
+
"load_all",
|
| 39 |
+
"dump",
|
| 40 |
+
"dump_all",
|
| 41 |
+
]
|
| 42 |
+
|
| 43 |
+
# ── Exceptions ─────────────────────────────────────────────────────────────────
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class YAMLError(Exception):
|
| 47 |
+
"""Raised when YAML parsing fails."""
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
# ── Scalar type resolution ─────────────────────────────────────────────────────
|
| 51 |
+
|
| 52 |
+
_NULL_RE = re.compile(r"\A(?:null|Null|NULL|~)\Z")
|
| 53 |
+
_BOOL_TRUE_RE = re.compile(r"\A(?:true|True|TRUE|yes|Yes|YES|on|On|ON)\Z")
|
| 54 |
+
_BOOL_FALSE_RE = re.compile(r"\A(?:false|False|FALSE|no|No|NO|off|Off|OFF)\Z")
|
| 55 |
+
_INT_RE = re.compile(r"\A[-+]?(?:0|[1-9][0-9_]*)\Z")
|
| 56 |
+
_INT_HEX_RE = re.compile(r"\A0x[0-9a-fA-F_]+\Z")
|
| 57 |
+
_INT_OCT_RE = re.compile(r"\A0[0-7_]+\Z") # YAML 1.1 octal: 0777 (not 0o777)
|
| 58 |
+
_INT_BIN_RE = re.compile(r"\A0b[01_]+\Z")
|
| 59 |
+
# YAML 1.1 requires explicit sign (+/-) in exponent when base has decimal point
|
| 60 |
+
_FLOAT_RE = re.compile(r"\A[-+]?(?:\.[0-9]+|[0-9]+\.[0-9]*)(?:[eE][-+][0-9]+)?\Z")
|
| 61 |
+
_INF_RE = re.compile(r"\A[-+]?\.(?:inf|Inf|INF)\Z")
|
| 62 |
+
_NAN_RE = re.compile(r"\A\.(?:nan|NaN|NAN)\Z")
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _resolve_scalar(value: str) -> str | int | float | bool | None:
|
| 66 |
+
"""Resolve a plain (unquoted) scalar string to a typed Python value."""
|
| 67 |
+
if not value:
|
| 68 |
+
return None
|
| 69 |
+
if _NULL_RE.match(value):
|
| 70 |
+
return None
|
| 71 |
+
if _BOOL_TRUE_RE.match(value):
|
| 72 |
+
return True
|
| 73 |
+
if _BOOL_FALSE_RE.match(value):
|
| 74 |
+
return False
|
| 75 |
+
if _INT_RE.match(value):
|
| 76 |
+
return int(value.replace("_", ""))
|
| 77 |
+
if _INT_HEX_RE.match(value):
|
| 78 |
+
return int(value.replace("_", ""), 16)
|
| 79 |
+
if _INT_OCT_RE.match(value):
|
| 80 |
+
return int(value.replace("_", ""), 8) # YAML 1.1: 0777 -> 511
|
| 81 |
+
if _INT_BIN_RE.match(value):
|
| 82 |
+
return int(value.replace("_", ""), 2)
|
| 83 |
+
if _FLOAT_RE.match(value):
|
| 84 |
+
return float(value.replace("_", ""))
|
| 85 |
+
if _INF_RE.match(value):
|
| 86 |
+
return float("-inf") if value.startswith("-") else float("inf")
|
| 87 |
+
if _NAN_RE.match(value):
|
| 88 |
+
return float("nan")
|
| 89 |
+
return value
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
# ── String unquoting ───────────────────────────────────────────────────────────
|
| 93 |
+
|
| 94 |
+
_DQ_ESCAPE_MAP = {
|
| 95 |
+
"\\": "\\",
|
| 96 |
+
'"': '"',
|
| 97 |
+
"n": "\n",
|
| 98 |
+
"r": "\r",
|
| 99 |
+
"t": "\t",
|
| 100 |
+
"0": "\0",
|
| 101 |
+
"a": "\a",
|
| 102 |
+
"b": "\b",
|
| 103 |
+
"e": "\x1b",
|
| 104 |
+
"v": "\v",
|
| 105 |
+
"/": "/",
|
| 106 |
+
" ": " ",
|
| 107 |
+
"N": "\x85",
|
| 108 |
+
"_": "\xa0",
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def _unescape_double_quoted(s: str) -> str:
|
| 113 |
+
"""Process escape sequences in a double-quoted YAML string."""
|
| 114 |
+
result: list[str] = []
|
| 115 |
+
i = 0
|
| 116 |
+
while i < len(s):
|
| 117 |
+
if s[i] == "\\" and i + 1 < len(s):
|
| 118 |
+
nxt = s[i + 1]
|
| 119 |
+
if nxt in _DQ_ESCAPE_MAP:
|
| 120 |
+
result.append(_DQ_ESCAPE_MAP[nxt])
|
| 121 |
+
i += 2
|
| 122 |
+
elif nxt == "x" and i + 3 < len(s):
|
| 123 |
+
result.append(chr(int(s[i + 2 : i + 4], 16)))
|
| 124 |
+
i += 4
|
| 125 |
+
elif nxt == "u" and i + 5 < len(s):
|
| 126 |
+
result.append(chr(int(s[i + 2 : i + 6], 16)))
|
| 127 |
+
i += 6
|
| 128 |
+
elif nxt == "U" and i + 9 < len(s):
|
| 129 |
+
result.append(chr(int(s[i + 2 : i + 10], 16)))
|
| 130 |
+
i += 10
|
| 131 |
+
else:
|
| 132 |
+
result.append(s[i])
|
| 133 |
+
i += 1
|
| 134 |
+
else:
|
| 135 |
+
result.append(s[i])
|
| 136 |
+
i += 1
|
| 137 |
+
return "".join(result)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _unquote(s: str) -> str:
|
| 141 |
+
"""Remove quotes from a YAML scalar and process escapes if needed."""
|
| 142 |
+
if len(s) >= 2:
|
| 143 |
+
if s[0] == "'" and s[-1] == "'":
|
| 144 |
+
# Single-quoted: only '' escapes to '
|
| 145 |
+
return s[1:-1].replace("''", "'")
|
| 146 |
+
if s[0] == '"' and s[-1] == '"':
|
| 147 |
+
return _unescape_double_quoted(s[1:-1])
|
| 148 |
+
return s
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
# ── Scanner ────────────────────────────────────────────────────────────────────
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
class _Line:
|
| 155 |
+
"""A logical line with tracked indentation and line number."""
|
| 156 |
+
|
| 157 |
+
__slots__ = ("indent", "text", "lineno")
|
| 158 |
+
|
| 159 |
+
def __init__(self, indent: int, text: str, lineno: int):
|
| 160 |
+
self.indent = indent
|
| 161 |
+
self.text = text
|
| 162 |
+
self.lineno = lineno
|
| 163 |
+
|
| 164 |
+
def __repr__(self) -> str:
|
| 165 |
+
return f"_Line({self.lineno}: indent={self.indent}, {self.text!r})"
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def _strip_inline_comment(text: str) -> str:
|
| 169 |
+
"""Remove inline comments from a line, respecting quoted strings."""
|
| 170 |
+
in_single = False
|
| 171 |
+
in_double = False
|
| 172 |
+
i = 0
|
| 173 |
+
while i < len(text):
|
| 174 |
+
ch = text[i]
|
| 175 |
+
if ch == "\\" and in_double and i + 1 < len(text):
|
| 176 |
+
i += 2
|
| 177 |
+
continue
|
| 178 |
+
if ch == "'" and not in_double:
|
| 179 |
+
in_single = not in_single
|
| 180 |
+
elif ch == '"' and not in_single:
|
| 181 |
+
in_double = not in_double
|
| 182 |
+
elif ch == "#" and not in_single and not in_double:
|
| 183 |
+
# Must be preceded by whitespace to be a comment
|
| 184 |
+
if i == 0 or text[i - 1] in (" ", "\t"):
|
| 185 |
+
return text[:i].rstrip()
|
| 186 |
+
i += 1
|
| 187 |
+
return text
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def _scan(text: str) -> list[_Line]:
|
| 191 |
+
"""Convert raw YAML text into a list of logical lines."""
|
| 192 |
+
lines: list[_Line] = []
|
| 193 |
+
for lineno, raw in enumerate(text.splitlines(), 1):
|
| 194 |
+
# Preserve completely empty lines for block scalar detection
|
| 195 |
+
stripped = raw.lstrip()
|
| 196 |
+
if not stripped or stripped[0] == "#":
|
| 197 |
+
continue
|
| 198 |
+
indent = len(raw) - len(stripped)
|
| 199 |
+
# Don't strip comments from block scalar indicators or document markers
|
| 200 |
+
if stripped.startswith("---") or stripped.startswith("..."):
|
| 201 |
+
lines.append(_Line(indent, stripped, lineno))
|
| 202 |
+
else:
|
| 203 |
+
cleaned = _strip_inline_comment(stripped)
|
| 204 |
+
if cleaned:
|
| 205 |
+
lines.append(_Line(indent, cleaned, lineno))
|
| 206 |
+
return lines
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
# ── Parser ─────────────────────────────────────────────────────────────────────
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
class _Parser:
|
| 213 |
+
"""Recursive-descent YAML parser for the common subset."""
|
| 214 |
+
|
| 215 |
+
def __init__(self, lines: list[_Line], raw_lines: list[str]):
|
| 216 |
+
self._lines = lines
|
| 217 |
+
self._raw_lines = raw_lines # original text lines for block scalars
|
| 218 |
+
self._pos = 0
|
| 219 |
+
|
| 220 |
+
def _peek(self) -> _Line | None:
|
| 221 |
+
if self._pos < len(self._lines):
|
| 222 |
+
return self._lines[self._pos]
|
| 223 |
+
return None
|
| 224 |
+
|
| 225 |
+
def _advance(self) -> _Line:
|
| 226 |
+
line = self._lines[self._pos]
|
| 227 |
+
self._pos += 1
|
| 228 |
+
return line
|
| 229 |
+
|
| 230 |
+
def _error(self, msg: str, lineno: int | None = None) -> YAMLError:
|
| 231 |
+
if lineno is None:
|
| 232 |
+
cur = self._peek()
|
| 233 |
+
lineno = cur.lineno if cur else -1
|
| 234 |
+
return YAMLError(f"line {lineno}: {msg}")
|
| 235 |
+
|
| 236 |
+
# ── Document parsing ──
|
| 237 |
+
|
| 238 |
+
def parse_stream(self) -> list[Any]:
|
| 239 |
+
"""Parse the entire stream, returning a list of documents."""
|
| 240 |
+
documents: list[Any] = []
|
| 241 |
+
|
| 242 |
+
# Skip leading document marker if present
|
| 243 |
+
cur = self._peek()
|
| 244 |
+
if cur is not None and cur.text == "---":
|
| 245 |
+
self._advance()
|
| 246 |
+
|
| 247 |
+
while self._pos < len(self._lines):
|
| 248 |
+
doc = self._parse_node(-1)
|
| 249 |
+
documents.append(doc)
|
| 250 |
+
|
| 251 |
+
# Check for document separator
|
| 252 |
+
cur = self._peek()
|
| 253 |
+
if cur is not None and cur.text in ("---", "..."):
|
| 254 |
+
self._advance()
|
| 255 |
+
# '...' without following '---' ends the stream
|
| 256 |
+
if cur.text == "...":
|
| 257 |
+
cur2 = self._peek()
|
| 258 |
+
if cur2 is None or cur2.text != "---":
|
| 259 |
+
break
|
| 260 |
+
self._advance() # skip the ---
|
| 261 |
+
|
| 262 |
+
if not documents:
|
| 263 |
+
documents.append(None)
|
| 264 |
+
|
| 265 |
+
return documents
|
| 266 |
+
|
| 267 |
+
def _parse_node(self, parent_indent: int) -> Any:
|
| 268 |
+
"""Parse a YAML node (mapping, sequence, or scalar)."""
|
| 269 |
+
cur = self._peek()
|
| 270 |
+
if cur is None:
|
| 271 |
+
return None
|
| 272 |
+
|
| 273 |
+
# Document end markers
|
| 274 |
+
if cur.text in ("---", "..."):
|
| 275 |
+
return None
|
| 276 |
+
|
| 277 |
+
text = cur.text
|
| 278 |
+
|
| 279 |
+
# Flow collections
|
| 280 |
+
if text.startswith("{"):
|
| 281 |
+
return self._parse_flow_mapping()
|
| 282 |
+
if text.startswith("["):
|
| 283 |
+
return self._parse_flow_sequence()
|
| 284 |
+
|
| 285 |
+
# Block scalar
|
| 286 |
+
if text.startswith("|") or text.startswith(">"):
|
| 287 |
+
return self._parse_block_scalar()
|
| 288 |
+
|
| 289 |
+
# Block sequence
|
| 290 |
+
if text.startswith("- ") or text == "-":
|
| 291 |
+
return self._parse_block_sequence(cur.indent)
|
| 292 |
+
|
| 293 |
+
# Check if it's a mapping (contains ': ' or ends with ':')
|
| 294 |
+
if self._is_mapping_line(text):
|
| 295 |
+
return self._parse_block_mapping(cur.indent)
|
| 296 |
+
|
| 297 |
+
# Plain scalar
|
| 298 |
+
self._advance()
|
| 299 |
+
return self._parse_scalar_value(text)
|
| 300 |
+
|
| 301 |
+
@staticmethod
|
| 302 |
+
def _skip_quoted_key(text: str) -> int:
|
| 303 |
+
"""Return the index after the quoted prefix of *text*.
|
| 304 |
+
|
| 305 |
+
If *text* starts with a single- or double-quoted string the returned
|
| 306 |
+
index points just past the closing quote. For unquoted text ``0`` is
|
| 307 |
+
returned so callers can scan from the beginning.
|
| 308 |
+
|
| 309 |
+
Returns ``-1`` when a single-quoted string has no closing quote (the
|
| 310 |
+
caller should treat this as "not a mapping line").
|
| 311 |
+
"""
|
| 312 |
+
if text.startswith("'"):
|
| 313 |
+
end = text.find("'", 1)
|
| 314 |
+
return -1 if end < 0 else end + 1
|
| 315 |
+
if text.startswith('"'):
|
| 316 |
+
i = 1
|
| 317 |
+
while i < len(text):
|
| 318 |
+
if text[i] == "\\" and i + 1 < len(text):
|
| 319 |
+
i += 2
|
| 320 |
+
continue
|
| 321 |
+
if text[i] == '"':
|
| 322 |
+
return i + 1
|
| 323 |
+
i += 1
|
| 324 |
+
return 0
|
| 325 |
+
|
| 326 |
+
@staticmethod
|
| 327 |
+
def _find_mapping_colon(text: str, start: int) -> int:
|
| 328 |
+
"""Find the position of the mapping separator colon starting from *start*.
|
| 329 |
+
|
| 330 |
+
Returns the index of the ``:`` that acts as a mapping separator, or
|
| 331 |
+
``-1`` if none is found. A colon qualifies when it is either the last
|
| 332 |
+
character or is immediately followed by a space or tab.
|
| 333 |
+
"""
|
| 334 |
+
i = start
|
| 335 |
+
while i < len(text):
|
| 336 |
+
if text[i] == ":":
|
| 337 |
+
if i + 1 == len(text) or text[i + 1] in (" ", "\t"):
|
| 338 |
+
return i
|
| 339 |
+
i += 1
|
| 340 |
+
return -1
|
| 341 |
+
|
| 342 |
+
def _is_mapping_line(self, text: str) -> bool:
|
| 343 |
+
"""Check if a line represents a mapping key."""
|
| 344 |
+
i = self._skip_quoted_key(text)
|
| 345 |
+
if i < 0:
|
| 346 |
+
return False
|
| 347 |
+
return self._find_mapping_colon(text, i) >= 0
|
| 348 |
+
|
| 349 |
+
def _split_mapping_line(self, text: str) -> tuple[str, str]:
|
| 350 |
+
"""Split a mapping line into key and value parts."""
|
| 351 |
+
i = self._skip_quoted_key(text)
|
| 352 |
+
if i < 0:
|
| 353 |
+
return text, ""
|
| 354 |
+
colon = self._find_mapping_colon(text, i)
|
| 355 |
+
if colon < 0:
|
| 356 |
+
return text, ""
|
| 357 |
+
if colon + 1 == len(text):
|
| 358 |
+
return text[:colon], ""
|
| 359 |
+
return text[:colon], text[colon + 2 :].strip()
|
| 360 |
+
|
| 361 |
+
# ── Block mapping ──
|
| 362 |
+
|
| 363 |
+
def _parse_inline_value(self, raw_value: str, lineno: int) -> Any:
|
| 364 |
+
"""Parse an inline mapping or sequence value."""
|
| 365 |
+
if raw_value.startswith(("{", "[")):
|
| 366 |
+
return self._parse_flow_from_text(raw_value)
|
| 367 |
+
if raw_value.startswith(("|", ">")):
|
| 368 |
+
return self._parse_block_scalar_from_indicator(raw_value, lineno)
|
| 369 |
+
return self._parse_scalar_value(raw_value)
|
| 370 |
+
|
| 371 |
+
def _at_block_end(self, cur: _Line | None, indent: int) -> bool:
|
| 372 |
+
"""Return True when the current line should stop a block collection."""
|
| 373 |
+
if cur is None or cur.indent < indent:
|
| 374 |
+
return True
|
| 375 |
+
return cur.text in ("---", "...") or cur.indent != indent
|
| 376 |
+
|
| 377 |
+
def _parse_block_mapping(self, indent: int) -> dict:
|
| 378 |
+
result: dict[Any, Any] = {}
|
| 379 |
+
while True:
|
| 380 |
+
cur = self._peek()
|
| 381 |
+
if self._at_block_end(cur, indent):
|
| 382 |
+
break
|
| 383 |
+
assert cur is not None # guaranteed by _at_block_end
|
| 384 |
+
if not self._is_mapping_line(cur.text):
|
| 385 |
+
break
|
| 386 |
+
|
| 387 |
+
self._advance()
|
| 388 |
+
raw_key, raw_value = self._split_mapping_line(cur.text)
|
| 389 |
+
key = self._parse_scalar_value(raw_key)
|
| 390 |
+
|
| 391 |
+
if raw_value:
|
| 392 |
+
value = self._parse_inline_value(raw_value, cur.lineno)
|
| 393 |
+
else:
|
| 394 |
+
nxt = self._peek()
|
| 395 |
+
if nxt is None or nxt.indent <= indent or nxt.text in ("---", "..."):
|
| 396 |
+
value = None
|
| 397 |
+
else:
|
| 398 |
+
value = self._parse_node(indent)
|
| 399 |
+
result[key] = value
|
| 400 |
+
return result
|
| 401 |
+
|
| 402 |
+
# ── Block sequence ──
|
| 403 |
+
|
| 404 |
+
def _parse_sequence_mapping_item(
|
| 405 |
+
self, item_text: str, cur: _Line, indent: int
|
| 406 |
+
) -> dict:
|
| 407 |
+
"""Parse a mapping that starts on a sequence item line (``- key: val``)."""
|
| 408 |
+
raw_key, raw_value = self._split_mapping_line(item_text)
|
| 409 |
+
key = self._parse_scalar_value(raw_key)
|
| 410 |
+
mapping: dict[Any, Any] = {}
|
| 411 |
+
|
| 412 |
+
if raw_value:
|
| 413 |
+
mapping[key] = self._parse_inline_value(raw_value, cur.lineno)
|
| 414 |
+
else:
|
| 415 |
+
nxt = self._peek()
|
| 416 |
+
if (
|
| 417 |
+
nxt is not None
|
| 418 |
+
and nxt.indent > indent
|
| 419 |
+
and nxt.text not in ("---", "...")
|
| 420 |
+
):
|
| 421 |
+
mapping[key] = self._parse_node(indent)
|
| 422 |
+
else:
|
| 423 |
+
mapping[key] = None
|
| 424 |
+
|
| 425 |
+
# Continue reading mapping entries at deeper indent
|
| 426 |
+
nxt = self._peek()
|
| 427 |
+
if nxt is not None and nxt.indent > indent and self._is_mapping_line(nxt.text):
|
| 428 |
+
rest = self._parse_block_mapping(nxt.indent)
|
| 429 |
+
mapping.update(rest)
|
| 430 |
+
|
| 431 |
+
return mapping
|
| 432 |
+
|
| 433 |
+
def _parse_sequence_item(self, item_text: str, cur: _Line, indent: int) -> Any:
|
| 434 |
+
"""Parse the value part of a single sequence item."""
|
| 435 |
+
if not item_text:
|
| 436 |
+
nxt = self._peek()
|
| 437 |
+
if nxt is None or nxt.indent <= indent or nxt.text in ("---", "..."):
|
| 438 |
+
return None
|
| 439 |
+
return self._parse_node(indent)
|
| 440 |
+
if item_text.startswith(("{", "[")):
|
| 441 |
+
return self._parse_flow_from_text(item_text)
|
| 442 |
+
if self._is_mapping_line(item_text):
|
| 443 |
+
return self._parse_sequence_mapping_item(item_text, cur, indent)
|
| 444 |
+
return self._parse_scalar_value(item_text)
|
| 445 |
+
|
| 446 |
+
def _parse_block_sequence(self, indent: int) -> list:
|
| 447 |
+
result: list[Any] = []
|
| 448 |
+
while True:
|
| 449 |
+
cur = self._peek()
|
| 450 |
+
if self._at_block_end(cur, indent):
|
| 451 |
+
break
|
| 452 |
+
assert cur is not None
|
| 453 |
+
if not (cur.text.startswith("- ") or cur.text == "-"):
|
| 454 |
+
break
|
| 455 |
+
|
| 456 |
+
self._advance()
|
| 457 |
+
item_text = cur.text[2:].strip() if cur.text.startswith("- ") else ""
|
| 458 |
+
result.append(self._parse_sequence_item(item_text, cur, indent))
|
| 459 |
+
|
| 460 |
+
return result
|
| 461 |
+
|
| 462 |
+
# ── Block scalars ──
|
| 463 |
+
|
| 464 |
+
def _parse_block_scalar(self) -> str:
|
| 465 |
+
cur = self._advance()
|
| 466 |
+
return self._parse_block_scalar_from_indicator(cur.text, cur.lineno)
|
| 467 |
+
|
| 468 |
+
@staticmethod
|
| 469 |
+
def _parse_chomping_indicator(header: str) -> tuple[str, int]:
|
| 470 |
+
"""Parse the block scalar header for chomping mode and explicit indent.
|
| 471 |
+
|
| 472 |
+
Args:
|
| 473 |
+
header: The portion of the indicator line after ``|`` or ``>``.
|
| 474 |
+
|
| 475 |
+
Returns:
|
| 476 |
+
A ``(chomp, explicit_indent)`` tuple.
|
| 477 |
+
"""
|
| 478 |
+
chomp = "clip"
|
| 479 |
+
explicit_indent = 0
|
| 480 |
+
for ch in header:
|
| 481 |
+
if ch == "-":
|
| 482 |
+
chomp = "strip"
|
| 483 |
+
elif ch == "+":
|
| 484 |
+
chomp = "keep"
|
| 485 |
+
elif ch.isdigit():
|
| 486 |
+
explicit_indent = int(ch)
|
| 487 |
+
return chomp, explicit_indent
|
| 488 |
+
|
| 489 |
+
def _collect_scalar_lines(self, raw_lineno: int, explicit_indent: int) -> list[str]:
|
| 490 |
+
"""Collect raw content lines for a block scalar.
|
| 491 |
+
|
| 492 |
+
Args:
|
| 493 |
+
raw_lineno: 1-based line number of the indicator (content starts
|
| 494 |
+
on the *next* raw line).
|
| 495 |
+
explicit_indent: Explicit indentation width from the header, or
|
| 496 |
+
``0`` to auto-detect.
|
| 497 |
+
|
| 498 |
+
Returns:
|
| 499 |
+
List of content lines with leading indentation stripped.
|
| 500 |
+
"""
|
| 501 |
+
if raw_lineno >= len(self._raw_lines):
|
| 502 |
+
return []
|
| 503 |
+
|
| 504 |
+
content_indent = self._detect_scalar_indent(raw_lineno, explicit_indent)
|
| 505 |
+
if content_indent == 0:
|
| 506 |
+
return []
|
| 507 |
+
|
| 508 |
+
content_lines: list[str] = []
|
| 509 |
+
for j in range(raw_lineno, len(self._raw_lines)):
|
| 510 |
+
raw = self._raw_lines[j]
|
| 511 |
+
if not raw.strip():
|
| 512 |
+
content_lines.append("")
|
| 513 |
+
continue
|
| 514 |
+
line_indent = len(raw) - len(raw.lstrip())
|
| 515 |
+
if line_indent < content_indent:
|
| 516 |
+
break
|
| 517 |
+
content_lines.append(raw[content_indent:])
|
| 518 |
+
return content_lines
|
| 519 |
+
|
| 520 |
+
def _detect_scalar_indent(self, raw_lineno: int, explicit: int) -> int:
|
| 521 |
+
"""Determine the content indentation for a block scalar."""
|
| 522 |
+
if explicit > 0:
|
| 523 |
+
return explicit
|
| 524 |
+
for j in range(raw_lineno, len(self._raw_lines)):
|
| 525 |
+
raw = self._raw_lines[j]
|
| 526 |
+
stripped = raw.lstrip()
|
| 527 |
+
if stripped and not stripped.startswith("#"):
|
| 528 |
+
return len(raw) - len(stripped)
|
| 529 |
+
return 0
|
| 530 |
+
|
| 531 |
+
@staticmethod
|
| 532 |
+
def _fold_lines(content_lines: list[str]) -> str:
|
| 533 |
+
"""Fold content lines (``>`` mode): replace single newlines with spaces."""
|
| 534 |
+
parts: list[str] = []
|
| 535 |
+
for line in content_lines:
|
| 536 |
+
if not line:
|
| 537 |
+
parts.append("\n")
|
| 538 |
+
elif parts and parts[-1] != "\n" and not parts[-1].endswith("\n"):
|
| 539 |
+
parts.append(" " + line)
|
| 540 |
+
else:
|
| 541 |
+
parts.append(line)
|
| 542 |
+
return "".join(parts)
|
| 543 |
+
|
| 544 |
+
@staticmethod
|
| 545 |
+
def _apply_chomping(text: str, chomp: str) -> str:
|
| 546 |
+
"""Apply the chomping rule (strip / keep / clip) to block scalar text."""
|
| 547 |
+
if chomp == "strip":
|
| 548 |
+
return text
|
| 549 |
+
if chomp == "keep":
|
| 550 |
+
return text + "\n"
|
| 551 |
+
# clip
|
| 552 |
+
return text + "\n" if text else ""
|
| 553 |
+
|
| 554 |
+
def _parse_block_scalar_from_indicator(
|
| 555 |
+
self, indicator_line: str, lineno: int
|
| 556 |
+
) -> str:
|
| 557 |
+
"""Parse a | or > block scalar, reading content lines from raw_lines."""
|
| 558 |
+
indicator = indicator_line[0]
|
| 559 |
+
header = indicator_line[1:].strip()
|
| 560 |
+
|
| 561 |
+
chomp, explicit_indent = self._parse_chomping_indicator(header)
|
| 562 |
+
content_lines = self._collect_scalar_lines(lineno, explicit_indent)
|
| 563 |
+
|
| 564 |
+
# Skip consumed content lines in the scanner
|
| 565 |
+
while True:
|
| 566 |
+
nxt = self._peek()
|
| 567 |
+
if nxt is None:
|
| 568 |
+
break
|
| 569 |
+
if nxt.lineno <= lineno + len(content_lines):
|
| 570 |
+
self._advance()
|
| 571 |
+
else:
|
| 572 |
+
break
|
| 573 |
+
|
| 574 |
+
# Remove trailing empty lines
|
| 575 |
+
while content_lines and not content_lines[-1]:
|
| 576 |
+
content_lines.pop()
|
| 577 |
+
|
| 578 |
+
if indicator == "|":
|
| 579 |
+
text = "\n".join(content_lines)
|
| 580 |
+
else:
|
| 581 |
+
text = self._fold_lines(content_lines)
|
| 582 |
+
|
| 583 |
+
return self._apply_chomping(text, chomp)
|
| 584 |
+
|
| 585 |
+
# ── Flow collections ──
|
| 586 |
+
|
| 587 |
+
def _parse_flow_mapping(self) -> dict:
|
| 588 |
+
cur = self._advance()
|
| 589 |
+
return self._parse_flow_from_text(cur.text)
|
| 590 |
+
|
| 591 |
+
def _parse_flow_sequence(self) -> list:
|
| 592 |
+
cur = self._advance()
|
| 593 |
+
return self._parse_flow_from_text(cur.text)
|
| 594 |
+
|
| 595 |
+
def _parse_flow_from_text(self, text: str) -> Any:
|
| 596 |
+
"""Parse a flow collection from raw text."""
|
| 597 |
+
tokens = _FlowTokenizer(text)
|
| 598 |
+
return tokens.parse()
|
| 599 |
+
|
| 600 |
+
# ── Scalar value ──
|
| 601 |
+
|
| 602 |
+
def _parse_scalar_value(self, text: str) -> Any:
|
| 603 |
+
"""Parse a scalar value, handling quoting and type resolution."""
|
| 604 |
+
if not text:
|
| 605 |
+
return None
|
| 606 |
+
# Quoted strings
|
| 607 |
+
if (text.startswith("'") and text.endswith("'")) or (
|
| 608 |
+
text.startswith('"') and text.endswith('"')
|
| 609 |
+
):
|
| 610 |
+
return _unquote(text)
|
| 611 |
+
# Plain scalar
|
| 612 |
+
return _resolve_scalar(text)
|
| 613 |
+
|
| 614 |
+
|
| 615 |
+
# ── Flow tokenizer ────────────────────────────────────────────────────────────
|
| 616 |
+
|
| 617 |
+
|
| 618 |
+
class _FlowTokenizer:
|
| 619 |
+
"""Character-level parser for flow-style YAML collections."""
|
| 620 |
+
|
| 621 |
+
def __init__(self, text: str):
|
| 622 |
+
self._text = text
|
| 623 |
+
self._pos = 0
|
| 624 |
+
|
| 625 |
+
def _skip_ws(self) -> None:
|
| 626 |
+
while self._pos < len(self._text) and self._text[self._pos] in (
|
| 627 |
+
" ",
|
| 628 |
+
"\t",
|
| 629 |
+
"\n",
|
| 630 |
+
"\r",
|
| 631 |
+
):
|
| 632 |
+
self._pos += 1
|
| 633 |
+
|
| 634 |
+
def _peek(self) -> str:
|
| 635 |
+
if self._pos < len(self._text):
|
| 636 |
+
return self._text[self._pos]
|
| 637 |
+
return ""
|
| 638 |
+
|
| 639 |
+
def parse(self) -> Any:
|
| 640 |
+
self._skip_ws()
|
| 641 |
+
ch = self._peek()
|
| 642 |
+
if ch == "{":
|
| 643 |
+
return self._parse_mapping()
|
| 644 |
+
if ch == "[":
|
| 645 |
+
return self._parse_sequence()
|
| 646 |
+
return self._parse_value()
|
| 647 |
+
|
| 648 |
+
def _parse_mapping(self) -> dict:
|
| 649 |
+
self._pos += 1 # skip {
|
| 650 |
+
result: dict[Any, Any] = {}
|
| 651 |
+
self._skip_ws()
|
| 652 |
+
if self._peek() == "}":
|
| 653 |
+
self._pos += 1
|
| 654 |
+
return result
|
| 655 |
+
|
| 656 |
+
while True:
|
| 657 |
+
self._skip_ws()
|
| 658 |
+
key = self._parse_value()
|
| 659 |
+
self._skip_ws()
|
| 660 |
+
if self._peek() == ":":
|
| 661 |
+
self._pos += 1
|
| 662 |
+
self._skip_ws()
|
| 663 |
+
value = self.parse()
|
| 664 |
+
else:
|
| 665 |
+
value = None
|
| 666 |
+
result[key] = value
|
| 667 |
+
self._skip_ws()
|
| 668 |
+
if self._peek() == ",":
|
| 669 |
+
self._pos += 1
|
| 670 |
+
self._skip_ws()
|
| 671 |
+
if self._peek() == "}":
|
| 672 |
+
self._pos += 1
|
| 673 |
+
break
|
| 674 |
+
elif self._peek() == "}":
|
| 675 |
+
self._pos += 1
|
| 676 |
+
break
|
| 677 |
+
else:
|
| 678 |
+
break
|
| 679 |
+
return result
|
| 680 |
+
|
| 681 |
+
def _parse_sequence(self) -> list:
|
| 682 |
+
self._pos += 1 # skip [
|
| 683 |
+
result: list[Any] = []
|
| 684 |
+
self._skip_ws()
|
| 685 |
+
if self._peek() == "]":
|
| 686 |
+
self._pos += 1
|
| 687 |
+
return result
|
| 688 |
+
|
| 689 |
+
while True:
|
| 690 |
+
self._skip_ws()
|
| 691 |
+
item = self.parse()
|
| 692 |
+
result.append(item)
|
| 693 |
+
self._skip_ws()
|
| 694 |
+
if self._peek() == ",":
|
| 695 |
+
self._pos += 1
|
| 696 |
+
self._skip_ws()
|
| 697 |
+
if self._peek() == "]":
|
| 698 |
+
self._pos += 1
|
| 699 |
+
break
|
| 700 |
+
elif self._peek() == "]":
|
| 701 |
+
self._pos += 1
|
| 702 |
+
break
|
| 703 |
+
else:
|
| 704 |
+
break
|
| 705 |
+
return result
|
| 706 |
+
|
| 707 |
+
def _parse_value(self) -> Any:
|
| 708 |
+
self._skip_ws()
|
| 709 |
+
ch = self._peek()
|
| 710 |
+
if ch == "{":
|
| 711 |
+
return self._parse_mapping()
|
| 712 |
+
if ch == "[":
|
| 713 |
+
return self._parse_sequence()
|
| 714 |
+
if ch == "'":
|
| 715 |
+
return self._parse_single_quoted()
|
| 716 |
+
if ch == '"':
|
| 717 |
+
return self._parse_double_quoted()
|
| 718 |
+
return self._parse_plain_scalar()
|
| 719 |
+
|
| 720 |
+
def _parse_single_quoted(self) -> str:
|
| 721 |
+
self._pos += 1 # skip opening '
|
| 722 |
+
parts: list[str] = []
|
| 723 |
+
while self._pos < len(self._text):
|
| 724 |
+
ch = self._text[self._pos]
|
| 725 |
+
if ch == "'":
|
| 726 |
+
if self._pos + 1 < len(self._text) and self._text[self._pos + 1] == "'":
|
| 727 |
+
parts.append("'")
|
| 728 |
+
self._pos += 2
|
| 729 |
+
else:
|
| 730 |
+
self._pos += 1
|
| 731 |
+
break
|
| 732 |
+
else:
|
| 733 |
+
parts.append(ch)
|
| 734 |
+
self._pos += 1
|
| 735 |
+
return "".join(parts)
|
| 736 |
+
|
| 737 |
+
def _parse_double_quoted(self) -> str:
|
| 738 |
+
self._pos += 1 # skip opening "
|
| 739 |
+
parts: list[str] = []
|
| 740 |
+
while self._pos < len(self._text):
|
| 741 |
+
ch = self._text[self._pos]
|
| 742 |
+
if ch == "\\" and self._pos + 1 < len(self._text):
|
| 743 |
+
nxt = self._text[self._pos + 1]
|
| 744 |
+
if nxt in _DQ_ESCAPE_MAP:
|
| 745 |
+
parts.append(_DQ_ESCAPE_MAP[nxt])
|
| 746 |
+
self._pos += 2
|
| 747 |
+
else:
|
| 748 |
+
parts.append(ch)
|
| 749 |
+
self._pos += 1
|
| 750 |
+
elif ch == '"':
|
| 751 |
+
self._pos += 1
|
| 752 |
+
break
|
| 753 |
+
else:
|
| 754 |
+
parts.append(ch)
|
| 755 |
+
self._pos += 1
|
| 756 |
+
return "".join(parts)
|
| 757 |
+
|
| 758 |
+
def _parse_plain_scalar(self) -> Any:
|
| 759 |
+
start = self._pos
|
| 760 |
+
# Read until a flow indicator or end
|
| 761 |
+
while self._pos < len(self._text):
|
| 762 |
+
ch = self._text[self._pos]
|
| 763 |
+
if ch in (",", "}", "]", "{", "[", ":"):
|
| 764 |
+
break
|
| 765 |
+
self._pos += 1
|
| 766 |
+
raw = self._text[start : self._pos].strip()
|
| 767 |
+
if not raw:
|
| 768 |
+
return None
|
| 769 |
+
return _resolve_scalar(raw)
|
| 770 |
+
|
| 771 |
+
|
| 772 |
+
# ── Dumper ─────────────────────────────────────────────────────────────────────
|
| 773 |
+
|
| 774 |
+
# Characters that force quoting in a plain scalar
|
| 775 |
+
_NEEDS_QUOTE_RE = re.compile(
|
| 776 |
+
r"[:\#{}\[\],&*?|>!%@`]"
|
| 777 |
+
r"|^[-?]$"
|
| 778 |
+
r"|^\s"
|
| 779 |
+
r"|\s$"
|
| 780 |
+
r"|\n"
|
| 781 |
+
)
|
| 782 |
+
|
| 783 |
+
|
| 784 |
+
class _Dumper:
|
| 785 |
+
"""YAML serializer."""
|
| 786 |
+
|
| 787 |
+
def __init__(
|
| 788 |
+
self,
|
| 789 |
+
indent: int = 2,
|
| 790 |
+
default_flow_style: bool | None = None,
|
| 791 |
+
sort_keys: bool = True,
|
| 792 |
+
allow_unicode: bool = True,
|
| 793 |
+
):
|
| 794 |
+
self._indent = indent
|
| 795 |
+
self._flow = default_flow_style
|
| 796 |
+
self._sort_keys = sort_keys
|
| 797 |
+
self._allow_unicode = allow_unicode
|
| 798 |
+
self._seen: set[int] = set()
|
| 799 |
+
|
| 800 |
+
def dump(self, data: Any) -> str:
|
| 801 |
+
self._seen.clear()
|
| 802 |
+
result = self._represent(data, 0)
|
| 803 |
+
if result.endswith("\n"):
|
| 804 |
+
return result
|
| 805 |
+
return result + "\n"
|
| 806 |
+
|
| 807 |
+
def _represent(self, data: Any, level: int) -> str:
|
| 808 |
+
if isinstance(data, dict):
|
| 809 |
+
return self._represent_mapping(data, level)
|
| 810 |
+
if isinstance(data, (list, tuple)):
|
| 811 |
+
return self._represent_sequence(data, level)
|
| 812 |
+
return self._represent_scalar(data)
|
| 813 |
+
|
| 814 |
+
def _represent_mapping(self, data: dict, level: int) -> str:
|
| 815 |
+
if id(data) in self._seen:
|
| 816 |
+
raise YAMLError("Circular reference detected")
|
| 817 |
+
self._seen.add(id(data))
|
| 818 |
+
|
| 819 |
+
if not data:
|
| 820 |
+
return "{}"
|
| 821 |
+
|
| 822 |
+
if self._flow is True:
|
| 823 |
+
items = []
|
| 824 |
+
keys = sorted(data.keys(), key=str) if self._sort_keys else data.keys()
|
| 825 |
+
for key in keys:
|
| 826 |
+
k = self._represent_scalar(key)
|
| 827 |
+
v = self._represent(data[key], level)
|
| 828 |
+
items.append(f"{k}: {v}")
|
| 829 |
+
self._seen.discard(id(data))
|
| 830 |
+
return "{" + ", ".join(items) + "}"
|
| 831 |
+
|
| 832 |
+
lines: list[str] = []
|
| 833 |
+
prefix = " " * (self._indent * level)
|
| 834 |
+
keys = sorted(data.keys(), key=str) if self._sort_keys else data.keys()
|
| 835 |
+
for key in keys:
|
| 836 |
+
k = self._represent_scalar(key)
|
| 837 |
+
val = data[key]
|
| 838 |
+
if isinstance(val, dict) and val:
|
| 839 |
+
lines.append(f"{prefix}{k}:")
|
| 840 |
+
lines.append(self._represent_mapping(val, level + 1))
|
| 841 |
+
elif isinstance(val, (list, tuple)) and val:
|
| 842 |
+
lines.append(f"{prefix}{k}:")
|
| 843 |
+
lines.append(self._represent_sequence(val, level + 1))
|
| 844 |
+
else:
|
| 845 |
+
v = self._represent(val, level + 1)
|
| 846 |
+
lines.append(f"{prefix}{k}: {v}")
|
| 847 |
+
self._seen.discard(id(data))
|
| 848 |
+
return "\n".join(lines)
|
| 849 |
+
|
| 850 |
+
def _represent_key_value_line(
|
| 851 |
+
self, key_str: str, val: Any, line_prefix: str, level: int
|
| 852 |
+
) -> list[str]:
|
| 853 |
+
"""Represent a single key-value pair for use inside a sequence item."""
|
| 854 |
+
if isinstance(val, (dict, list, tuple)) and val:
|
| 855 |
+
return [f"{line_prefix}{key_str}:", self._represent(val, level)]
|
| 856 |
+
v = self._represent(val, level)
|
| 857 |
+
return [f"{line_prefix}{key_str}: {v}"]
|
| 858 |
+
|
| 859 |
+
def _represent_dict_in_sequence(
|
| 860 |
+
self, item: dict, prefix: str, level: int
|
| 861 |
+
) -> list[str]:
|
| 862 |
+
"""Represent a dict that appears as a sequence item (``- key: val``)."""
|
| 863 |
+
keys = sorted(item.keys(), key=str) if self._sort_keys else list(item.keys())
|
| 864 |
+
first_key = keys[0]
|
| 865 |
+
k = self._represent_scalar(first_key)
|
| 866 |
+
lines = self._represent_key_value_line(
|
| 867 |
+
k, item[first_key], f"{prefix}- ", level + 2
|
| 868 |
+
)
|
| 869 |
+
inner_prefix = prefix + " " * self._indent
|
| 870 |
+
for rk in keys[1:]:
|
| 871 |
+
rk_s = self._represent_scalar(rk)
|
| 872 |
+
lines.extend(
|
| 873 |
+
self._represent_key_value_line(rk_s, item[rk], inner_prefix, level + 2)
|
| 874 |
+
)
|
| 875 |
+
return lines
|
| 876 |
+
|
| 877 |
+
def _represent_sequence(self, data: list | tuple, level: int) -> str:
|
| 878 |
+
if id(data) in self._seen:
|
| 879 |
+
raise YAMLError("Circular reference detected")
|
| 880 |
+
self._seen.add(id(data))
|
| 881 |
+
|
| 882 |
+
if not data:
|
| 883 |
+
return "[]"
|
| 884 |
+
|
| 885 |
+
if self._flow is True:
|
| 886 |
+
items = [self._represent(item, level) for item in data]
|
| 887 |
+
self._seen.discard(id(data))
|
| 888 |
+
return "[" + ", ".join(items) + "]"
|
| 889 |
+
|
| 890 |
+
lines: list[str] = []
|
| 891 |
+
prefix = " " * (self._indent * level)
|
| 892 |
+
for item in data:
|
| 893 |
+
if isinstance(item, dict) and item:
|
| 894 |
+
lines.extend(self._represent_dict_in_sequence(item, prefix, level))
|
| 895 |
+
elif isinstance(item, (list, tuple)) and item:
|
| 896 |
+
lines.append(f"{prefix}-")
|
| 897 |
+
lines.append(self._represent_sequence(item, level + 1))
|
| 898 |
+
else:
|
| 899 |
+
v = self._represent(item, level + 1)
|
| 900 |
+
lines.append(f"{prefix}- {v}")
|
| 901 |
+
|
| 902 |
+
self._seen.discard(id(data))
|
| 903 |
+
return "\n".join(lines)
|
| 904 |
+
|
| 905 |
+
def _represent_scalar(self, data: Any) -> str:
|
| 906 |
+
if data is None:
|
| 907 |
+
return "null"
|
| 908 |
+
if isinstance(data, bool):
|
| 909 |
+
return "true" if data else "false"
|
| 910 |
+
if isinstance(data, int):
|
| 911 |
+
return str(data)
|
| 912 |
+
if isinstance(data, float):
|
| 913 |
+
if math.isnan(data):
|
| 914 |
+
return ".nan"
|
| 915 |
+
if math.isinf(data):
|
| 916 |
+
return ".inf" if data > 0 else "-.inf"
|
| 917 |
+
return str(data)
|
| 918 |
+
if isinstance(data, str):
|
| 919 |
+
return self._represent_str(data)
|
| 920 |
+
return str(data)
|
| 921 |
+
|
| 922 |
+
def _represent_str(self, s: str) -> str:
|
| 923 |
+
if not s:
|
| 924 |
+
return "''"
|
| 925 |
+
|
| 926 |
+
# Check if it looks like a special YAML value
|
| 927 |
+
if (
|
| 928 |
+
_NULL_RE.match(s)
|
| 929 |
+
or _BOOL_TRUE_RE.match(s)
|
| 930 |
+
or _BOOL_FALSE_RE.match(s)
|
| 931 |
+
or _INT_RE.match(s)
|
| 932 |
+
or _INT_HEX_RE.match(s)
|
| 933 |
+
or _FLOAT_RE.match(s)
|
| 934 |
+
or _INF_RE.match(s)
|
| 935 |
+
or _NAN_RE.match(s)
|
| 936 |
+
):
|
| 937 |
+
return f"'{s}'"
|
| 938 |
+
|
| 939 |
+
# Check if quoting is needed
|
| 940 |
+
if _NEEDS_QUOTE_RE.search(s):
|
| 941 |
+
if "\n" in s:
|
| 942 |
+
# Use literal block scalar for multiline
|
| 943 |
+
return "|\n" + "\n".join(" " + line for line in s.split("\n"))
|
| 944 |
+
# Use single quotes, escaping internal single quotes
|
| 945 |
+
return "'" + s.replace("'", "''") + "'"
|
| 946 |
+
|
| 947 |
+
return s
|
| 948 |
+
|
| 949 |
+
|
| 950 |
+
# ── Public API ─────────────────────────────────────────────────────────────────
|
| 951 |
+
|
| 952 |
+
|
| 953 |
+
def load(text: str) -> Any:
|
| 954 |
+
"""Parse a YAML string and return a Python object.
|
| 955 |
+
|
| 956 |
+
Only produces safe types: dict, list, str, int, float, bool, None.
|
| 957 |
+
Equivalent to PyYAML's ``yaml.safe_load()``.
|
| 958 |
+
|
| 959 |
+
Args:
|
| 960 |
+
text: YAML document string.
|
| 961 |
+
|
| 962 |
+
Returns:
|
| 963 |
+
Parsed Python object.
|
| 964 |
+
|
| 965 |
+
Raises:
|
| 966 |
+
YAMLError: If the YAML is malformed.
|
| 967 |
+
"""
|
| 968 |
+
if not text or not text.strip():
|
| 969 |
+
return None
|
| 970 |
+
raw_lines = text.splitlines()
|
| 971 |
+
lines = _scan(text)
|
| 972 |
+
if not lines:
|
| 973 |
+
return None
|
| 974 |
+
parser = _Parser(lines, raw_lines)
|
| 975 |
+
docs = parser.parse_stream()
|
| 976 |
+
return docs[0] if docs else None
|
| 977 |
+
|
| 978 |
+
|
| 979 |
+
def load_all(text: str) -> Iterator[Any]:
|
| 980 |
+
"""Parse a multi-document YAML string.
|
| 981 |
+
|
| 982 |
+
Yields one Python object per YAML document (separated by ``---``).
|
| 983 |
+
|
| 984 |
+
Args:
|
| 985 |
+
text: Multi-document YAML string.
|
| 986 |
+
|
| 987 |
+
Yields:
|
| 988 |
+
Parsed Python objects, one per document.
|
| 989 |
+
"""
|
| 990 |
+
if not text or not text.strip():
|
| 991 |
+
yield None
|
| 992 |
+
return
|
| 993 |
+
raw_lines = text.splitlines()
|
| 994 |
+
lines = _scan(text)
|
| 995 |
+
if not lines:
|
| 996 |
+
yield None
|
| 997 |
+
return
|
| 998 |
+
parser = _Parser(lines, raw_lines)
|
| 999 |
+
docs = parser.parse_stream()
|
| 1000 |
+
yield from docs
|
| 1001 |
+
|
| 1002 |
+
|
| 1003 |
+
@overload
|
| 1004 |
+
def dump(
|
| 1005 |
+
data: Any,
|
| 1006 |
+
stream: None = None,
|
| 1007 |
+
*,
|
| 1008 |
+
default_flow_style: bool | None = None,
|
| 1009 |
+
indent: int = 2,
|
| 1010 |
+
sort_keys: bool = True,
|
| 1011 |
+
allow_unicode: bool = True,
|
| 1012 |
+
) -> str: ...
|
| 1013 |
+
|
| 1014 |
+
|
| 1015 |
+
@overload
|
| 1016 |
+
def dump(
|
| 1017 |
+
data: Any,
|
| 1018 |
+
stream: IO[str],
|
| 1019 |
+
*,
|
| 1020 |
+
default_flow_style: bool | None = None,
|
| 1021 |
+
indent: int = 2,
|
| 1022 |
+
sort_keys: bool = True,
|
| 1023 |
+
allow_unicode: bool = True,
|
| 1024 |
+
) -> None: ...
|
| 1025 |
+
|
| 1026 |
+
|
| 1027 |
+
def dump(
|
| 1028 |
+
data: Any,
|
| 1029 |
+
stream: IO[str] | None = None,
|
| 1030 |
+
*,
|
| 1031 |
+
default_flow_style: bool | None = None,
|
| 1032 |
+
indent: int = 2,
|
| 1033 |
+
sort_keys: bool = True,
|
| 1034 |
+
allow_unicode: bool = True,
|
| 1035 |
+
) -> str | None:
|
| 1036 |
+
"""Serialize a Python object to a YAML string.
|
| 1037 |
+
|
| 1038 |
+
Args:
|
| 1039 |
+
data: Python object to serialize.
|
| 1040 |
+
stream: If provided, write to this stream and return None.
|
| 1041 |
+
default_flow_style: True for flow (inline) style, False for block,
|
| 1042 |
+
None for auto (empty collections use flow).
|
| 1043 |
+
indent: Number of spaces per indentation level.
|
| 1044 |
+
sort_keys: Sort mapping keys alphabetically.
|
| 1045 |
+
allow_unicode: Allow unicode characters in output.
|
| 1046 |
+
|
| 1047 |
+
Returns:
|
| 1048 |
+
YAML string if *stream* is None, otherwise None.
|
| 1049 |
+
"""
|
| 1050 |
+
dumper = _Dumper(
|
| 1051 |
+
indent=indent,
|
| 1052 |
+
default_flow_style=default_flow_style,
|
| 1053 |
+
sort_keys=sort_keys,
|
| 1054 |
+
allow_unicode=allow_unicode,
|
| 1055 |
+
)
|
| 1056 |
+
result = dumper.dump(data)
|
| 1057 |
+
if stream is not None:
|
| 1058 |
+
stream.write(result)
|
| 1059 |
+
return None
|
| 1060 |
+
return result
|
| 1061 |
+
|
| 1062 |
+
|
| 1063 |
+
@overload
|
| 1064 |
+
def dump_all(
|
| 1065 |
+
documents: list[Any] | tuple[Any, ...],
|
| 1066 |
+
stream: None = None,
|
| 1067 |
+
*,
|
| 1068 |
+
default_flow_style: bool | None = None,
|
| 1069 |
+
indent: int = 2,
|
| 1070 |
+
sort_keys: bool = True,
|
| 1071 |
+
allow_unicode: bool = True,
|
| 1072 |
+
) -> str: ...
|
| 1073 |
+
|
| 1074 |
+
|
| 1075 |
+
@overload
|
| 1076 |
+
def dump_all(
|
| 1077 |
+
documents: list[Any] | tuple[Any, ...],
|
| 1078 |
+
stream: IO[str],
|
| 1079 |
+
*,
|
| 1080 |
+
default_flow_style: bool | None = None,
|
| 1081 |
+
indent: int = 2,
|
| 1082 |
+
sort_keys: bool = True,
|
| 1083 |
+
allow_unicode: bool = True,
|
| 1084 |
+
) -> None: ...
|
| 1085 |
+
|
| 1086 |
+
|
| 1087 |
+
def dump_all(
|
| 1088 |
+
documents: list[Any] | tuple[Any, ...],
|
| 1089 |
+
stream: IO[str] | None = None,
|
| 1090 |
+
*,
|
| 1091 |
+
default_flow_style: bool | None = None,
|
| 1092 |
+
indent: int = 2,
|
| 1093 |
+
sort_keys: bool = True,
|
| 1094 |
+
allow_unicode: bool = True,
|
| 1095 |
+
) -> str | None:
|
| 1096 |
+
"""Serialize multiple Python objects as a multi-document YAML string.
|
| 1097 |
+
|
| 1098 |
+
Args:
|
| 1099 |
+
documents: Iterable of Python objects to serialize.
|
| 1100 |
+
stream: If provided, write to this stream and return None.
|
| 1101 |
+
default_flow_style: True for flow style, False for block, None for auto.
|
| 1102 |
+
indent: Number of spaces per indentation level.
|
| 1103 |
+
sort_keys: Sort mapping keys alphabetically.
|
| 1104 |
+
allow_unicode: Allow unicode characters in output.
|
| 1105 |
+
|
| 1106 |
+
Returns:
|
| 1107 |
+
YAML string if *stream* is None, otherwise None.
|
| 1108 |
+
"""
|
| 1109 |
+
dumper = _Dumper(
|
| 1110 |
+
indent=indent,
|
| 1111 |
+
default_flow_style=default_flow_style,
|
| 1112 |
+
sort_keys=sort_keys,
|
| 1113 |
+
allow_unicode=allow_unicode,
|
| 1114 |
+
)
|
| 1115 |
+
parts: list[str] = []
|
| 1116 |
+
for i, doc in enumerate(documents):
|
| 1117 |
+
if i > 0:
|
| 1118 |
+
parts.append("---\n")
|
| 1119 |
+
parts.append(dumper.dump(doc))
|
| 1120 |
+
result = "".join(parts)
|
| 1121 |
+
if stream is not None:
|
| 1122 |
+
stream.write(result)
|
| 1123 |
+
return None
|
| 1124 |
+
return result
|
|
@@ -0,0 +1,158 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""VeilRender application entry point."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import asyncio
|
| 6 |
+
import logging
|
| 7 |
+
import signal
|
| 8 |
+
import sys
|
| 9 |
+
|
| 10 |
+
from veilrender._vendor.httpserver import App
|
| 11 |
+
from veilrender.browser import browser_manager
|
| 12 |
+
from veilrender.cdp_proxy import handle_cdp_upgrade, is_websocket_upgrade
|
| 13 |
+
from veilrender.config import settings
|
| 14 |
+
from veilrender.routes import health, render, screenshot
|
| 15 |
+
|
| 16 |
+
logging.basicConfig(
|
| 17 |
+
level=logging.INFO,
|
| 18 |
+
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
| 19 |
+
)
|
| 20 |
+
logger = logging.getLogger(__name__)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
async def _pipe(source: asyncio.StreamReader, dest: asyncio.StreamReader) -> None:
|
| 24 |
+
"""Relay remaining data from *source* into *dest* until EOF."""
|
| 25 |
+
try:
|
| 26 |
+
while True:
|
| 27 |
+
chunk = await source.read(8192)
|
| 28 |
+
if not chunk:
|
| 29 |
+
break
|
| 30 |
+
dest.feed_data(chunk)
|
| 31 |
+
except Exception:
|
| 32 |
+
pass
|
| 33 |
+
finally:
|
| 34 |
+
dest.feed_eof()
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def create_app() -> App:
|
| 38 |
+
"""Create and configure the VeilRender application."""
|
| 39 |
+
app = App(max_body_size=10 * 1024 * 1024) # 10 MB
|
| 40 |
+
|
| 41 |
+
# Register routes
|
| 42 |
+
health.register(app)
|
| 43 |
+
render.register(app)
|
| 44 |
+
screenshot.register(app)
|
| 45 |
+
|
| 46 |
+
return app
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def main() -> None:
|
| 50 |
+
"""Run the VeilRender server with HTTP + CDP WebSocket multiplexing."""
|
| 51 |
+
app = create_app()
|
| 52 |
+
|
| 53 |
+
async def run_server() -> None:
|
| 54 |
+
await browser_manager.start()
|
| 55 |
+
|
| 56 |
+
shutdown_event = asyncio.Event()
|
| 57 |
+
|
| 58 |
+
async def handle_connection(
|
| 59 |
+
reader: asyncio.StreamReader, writer: asyncio.StreamWriter
|
| 60 |
+
) -> None:
|
| 61 |
+
"""Multiplex: WebSocket upgrade to /cdp → CDP proxy, else → HTTP."""
|
| 62 |
+
try:
|
| 63 |
+
# Peek at the first request to decide routing
|
| 64 |
+
raw = b""
|
| 65 |
+
while b"\r\n\r\n" not in raw:
|
| 66 |
+
chunk = await asyncio.wait_for(reader.read(8192), timeout=30)
|
| 67 |
+
if not chunk:
|
| 68 |
+
writer.close()
|
| 69 |
+
return
|
| 70 |
+
raw += chunk
|
| 71 |
+
|
| 72 |
+
# Parse the request line and headers
|
| 73 |
+
header_block, _, body_start = raw.partition(b"\r\n\r\n")
|
| 74 |
+
lines = header_block.decode("latin-1").split("\r\n")
|
| 75 |
+
request_line = lines[0]
|
| 76 |
+
parts = request_line.split(" ", 2)
|
| 77 |
+
if len(parts) < 2:
|
| 78 |
+
writer.close()
|
| 79 |
+
return
|
| 80 |
+
|
| 81 |
+
method = parts[0].upper()
|
| 82 |
+
raw_path = parts[1]
|
| 83 |
+
|
| 84 |
+
# Parse path and query string
|
| 85 |
+
if "?" in raw_path:
|
| 86 |
+
path, query_string = raw_path.split("?", 1)
|
| 87 |
+
else:
|
| 88 |
+
path, query_string = raw_path, ""
|
| 89 |
+
|
| 90 |
+
# Parse headers
|
| 91 |
+
headers: dict[str, str] = {}
|
| 92 |
+
for line in lines[1:]:
|
| 93 |
+
if ":" in line:
|
| 94 |
+
key, _, value = line.partition(":")
|
| 95 |
+
headers[key.strip().lower()] = value.strip()
|
| 96 |
+
|
| 97 |
+
# Check if this is a WebSocket upgrade to /cdp
|
| 98 |
+
if is_websocket_upgrade(method, headers, path):
|
| 99 |
+
await handle_cdp_upgrade(
|
| 100 |
+
reader,
|
| 101 |
+
writer,
|
| 102 |
+
headers,
|
| 103 |
+
path,
|
| 104 |
+
query_string,
|
| 105 |
+
browser_manager.get_cdp_url,
|
| 106 |
+
)
|
| 107 |
+
return
|
| 108 |
+
|
| 109 |
+
# Otherwise, delegate to httpserver.
|
| 110 |
+
# Re-feed the buffered bytes and pipe any remaining body
|
| 111 |
+
# data from the original reader so POST bodies are not
|
| 112 |
+
# truncated (see #5).
|
| 113 |
+
combined_reader = asyncio.StreamReader()
|
| 114 |
+
combined_reader.feed_data(raw)
|
| 115 |
+
pipe_task = asyncio.create_task(_pipe(reader, combined_reader))
|
| 116 |
+
try:
|
| 117 |
+
await app._handle_connection(combined_reader, writer)
|
| 118 |
+
finally:
|
| 119 |
+
pipe_task.cancel()
|
| 120 |
+
except Exception:
|
| 121 |
+
logger.debug("Connection handler error", exc_info=True)
|
| 122 |
+
try:
|
| 123 |
+
writer.close()
|
| 124 |
+
except Exception:
|
| 125 |
+
pass
|
| 126 |
+
|
| 127 |
+
server = await asyncio.start_server(
|
| 128 |
+
handle_connection, settings.host, settings.port
|
| 129 |
+
)
|
| 130 |
+
addr = (
|
| 131 |
+
server.sockets[0].getsockname()
|
| 132 |
+
if server.sockets
|
| 133 |
+
else (settings.host, settings.port)
|
| 134 |
+
)
|
| 135 |
+
logger.info(
|
| 136 |
+
"VeilRender serving on %s:%d (auth=%s, cdp=ws://%s:%d/cdp)",
|
| 137 |
+
addr[0],
|
| 138 |
+
addr[1],
|
| 139 |
+
"enabled" if settings.api_token else "disabled",
|
| 140 |
+
addr[0],
|
| 141 |
+
addr[1],
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
loop = asyncio.get_running_loop()
|
| 145 |
+
if sys.platform != "win32":
|
| 146 |
+
for sig in (signal.SIGINT, signal.SIGTERM):
|
| 147 |
+
loop.add_signal_handler(sig, shutdown_event.set)
|
| 148 |
+
|
| 149 |
+
try:
|
| 150 |
+
async with server:
|
| 151 |
+
await shutdown_event.wait()
|
| 152 |
+
finally:
|
| 153 |
+
await browser_manager.stop()
|
| 154 |
+
|
| 155 |
+
try:
|
| 156 |
+
asyncio.run(run_server())
|
| 157 |
+
except KeyboardInterrupt:
|
| 158 |
+
pass
|
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Token-based authentication."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from veilrender._vendor.httpserver import HTTPException, Request
|
| 6 |
+
from veilrender.config import settings
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def verify_token(request: Request) -> None:
|
| 10 |
+
"""Verify the API token from header or query param.
|
| 11 |
+
|
| 12 |
+
Checks ``Authorization: Bearer <token>`` header first, then falls
|
| 13 |
+
back to ``?token=<token>`` query parameter.
|
| 14 |
+
|
| 15 |
+
Raises:
|
| 16 |
+
HTTPException: 401 if token is invalid, 403 if token is missing.
|
| 17 |
+
|
| 18 |
+
If ``VEILRENDER_API_TOKEN`` is not configured, auth is disabled.
|
| 19 |
+
"""
|
| 20 |
+
expected = settings.api_token
|
| 21 |
+
if expected is None:
|
| 22 |
+
return
|
| 23 |
+
|
| 24 |
+
# Check Authorization header
|
| 25 |
+
auth_header = request.headers.get("authorization", "")
|
| 26 |
+
if auth_header.startswith("Bearer "):
|
| 27 |
+
token = auth_header[7:].strip()
|
| 28 |
+
if token == expected:
|
| 29 |
+
return
|
| 30 |
+
raise HTTPException(401, "Invalid token")
|
| 31 |
+
|
| 32 |
+
# Check query param
|
| 33 |
+
token_params = request.query_params.get("token", [])
|
| 34 |
+
if token_params:
|
| 35 |
+
if token_params[0] == expected:
|
| 36 |
+
return
|
| 37 |
+
raise HTTPException(401, "Invalid token")
|
| 38 |
+
|
| 39 |
+
raise HTTPException(403, "Authentication required")
|
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Playwright browser lifecycle management."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import asyncio
|
| 6 |
+
import logging
|
| 7 |
+
import subprocess
|
| 8 |
+
from contextlib import asynccontextmanager
|
| 9 |
+
from collections.abc import AsyncIterator
|
| 10 |
+
|
| 11 |
+
from cloakbrowser import ensure_binary, get_default_stealth_args
|
| 12 |
+
from playwright.async_api import Browser, BrowserContext, Page, async_playwright
|
| 13 |
+
|
| 14 |
+
from veilrender.config import settings
|
| 15 |
+
|
| 16 |
+
logger = logging.getLogger(__name__)
|
| 17 |
+
|
| 18 |
+
CDP_PORT = 9222
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class BrowserManager:
|
| 22 |
+
"""Manages a shared Playwright browser instance.
|
| 23 |
+
|
| 24 |
+
Launches Chromium directly (not via Playwright's launch()) so that
|
| 25 |
+
``--remote-debugging-port`` actually takes effect. Playwright then
|
| 26 |
+
connects over CDP, and the CDP port is also available for external
|
| 27 |
+
clients via the WebSocket proxy.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
def __init__(self) -> None:
|
| 31 |
+
self._playwright = None
|
| 32 |
+
self._browser: Browser | None = None
|
| 33 |
+
self._chrome_proc: subprocess.Popen | None = None # type: ignore[type-arg]
|
| 34 |
+
self._semaphore = asyncio.Semaphore(settings.max_concurrent)
|
| 35 |
+
self._lock = asyncio.Lock()
|
| 36 |
+
|
| 37 |
+
async def start(self) -> None:
|
| 38 |
+
"""Launch Chromium with CDP and connect Playwright to it."""
|
| 39 |
+
executable_path = ensure_binary()
|
| 40 |
+
stealth_args = get_default_stealth_args()
|
| 41 |
+
|
| 42 |
+
chrome_args = [
|
| 43 |
+
executable_path,
|
| 44 |
+
"--headless",
|
| 45 |
+
f"--remote-debugging-port={CDP_PORT}",
|
| 46 |
+
"--no-first-run",
|
| 47 |
+
"--no-default-browser-check",
|
| 48 |
+
"--disable-setuid-sandbox",
|
| 49 |
+
"--disable-dev-shm-usage",
|
| 50 |
+
"--disable-gpu",
|
| 51 |
+
*stealth_args,
|
| 52 |
+
"about:blank",
|
| 53 |
+
]
|
| 54 |
+
|
| 55 |
+
self._chrome_proc = subprocess.Popen(
|
| 56 |
+
chrome_args,
|
| 57 |
+
stdout=subprocess.DEVNULL,
|
| 58 |
+
stderr=subprocess.PIPE,
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
# Wait for CDP to be ready
|
| 62 |
+
cdp_url = f"http://127.0.0.1:{CDP_PORT}"
|
| 63 |
+
for attempt in range(30):
|
| 64 |
+
try:
|
| 65 |
+
import urllib.request
|
| 66 |
+
|
| 67 |
+
urllib.request.urlopen(f"{cdp_url}/json/version", timeout=1)
|
| 68 |
+
break
|
| 69 |
+
except Exception:
|
| 70 |
+
await asyncio.sleep(0.5)
|
| 71 |
+
else:
|
| 72 |
+
stderr = (
|
| 73 |
+
self._chrome_proc.stderr.read().decode()
|
| 74 |
+
if self._chrome_proc.stderr
|
| 75 |
+
else ""
|
| 76 |
+
)
|
| 77 |
+
raise RuntimeError(
|
| 78 |
+
f"Chromium CDP not ready after 15s. stderr: {stderr[:500]}"
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
# Connect Playwright over CDP
|
| 82 |
+
self._playwright = await async_playwright().start()
|
| 83 |
+
self._browser = await self._playwright.chromium.connect_over_cdp(
|
| 84 |
+
f"http://127.0.0.1:{CDP_PORT}"
|
| 85 |
+
)
|
| 86 |
+
logger.info(
|
| 87 |
+
"Browser started (CloakBrowser %s, CDP on :%d)", executable_path, CDP_PORT
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
async def get_cdp_url(self) -> str | None:
|
| 91 |
+
"""Return the internal CDP WebSocket URL, or None if unavailable."""
|
| 92 |
+
await self._ensure_browser()
|
| 93 |
+
try:
|
| 94 |
+
import urllib.request
|
| 95 |
+
import json
|
| 96 |
+
|
| 97 |
+
resp = urllib.request.urlopen(
|
| 98 |
+
f"http://127.0.0.1:{CDP_PORT}/json/version", timeout=2
|
| 99 |
+
)
|
| 100 |
+
data = json.loads(resp.read())
|
| 101 |
+
ws_url = data.get("webSocketDebuggerUrl")
|
| 102 |
+
if ws_url:
|
| 103 |
+
return ws_url
|
| 104 |
+
except Exception:
|
| 105 |
+
logger.debug("Failed to get CDP WebSocket URL", exc_info=True)
|
| 106 |
+
return f"ws://127.0.0.1:{CDP_PORT}"
|
| 107 |
+
|
| 108 |
+
async def stop(self) -> None:
|
| 109 |
+
"""Close the browser, Playwright, and Chromium process."""
|
| 110 |
+
if self._browser:
|
| 111 |
+
try:
|
| 112 |
+
await self._browser.close()
|
| 113 |
+
except Exception:
|
| 114 |
+
pass
|
| 115 |
+
self._browser = None
|
| 116 |
+
if self._playwright:
|
| 117 |
+
await self._playwright.stop()
|
| 118 |
+
self._playwright = None
|
| 119 |
+
if self._chrome_proc:
|
| 120 |
+
self._chrome_proc.terminate()
|
| 121 |
+
try:
|
| 122 |
+
self._chrome_proc.wait(timeout=5)
|
| 123 |
+
except subprocess.TimeoutExpired:
|
| 124 |
+
self._chrome_proc.kill()
|
| 125 |
+
self._chrome_proc = None
|
| 126 |
+
logger.info("Browser stopped")
|
| 127 |
+
|
| 128 |
+
async def _ensure_browser(self) -> Browser:
|
| 129 |
+
"""Restart browser if it crashed."""
|
| 130 |
+
async with self._lock:
|
| 131 |
+
chrome_dead = (
|
| 132 |
+
self._chrome_proc is None or self._chrome_proc.poll() is not None
|
| 133 |
+
)
|
| 134 |
+
browser_dead = self._browser is None or not self._browser.is_connected()
|
| 135 |
+
if chrome_dead or browser_dead:
|
| 136 |
+
logger.warning("Browser not connected, restarting...")
|
| 137 |
+
await self.stop()
|
| 138 |
+
await self.start()
|
| 139 |
+
assert self._browser is not None
|
| 140 |
+
return self._browser
|
| 141 |
+
|
| 142 |
+
@asynccontextmanager
|
| 143 |
+
async def get_page(
|
| 144 |
+
self,
|
| 145 |
+
*,
|
| 146 |
+
viewport_width: int | None = None,
|
| 147 |
+
viewport_height: int | None = None,
|
| 148 |
+
) -> AsyncIterator[tuple[BrowserContext, Page]]:
|
| 149 |
+
"""Create an isolated browser context and page.
|
| 150 |
+
|
| 151 |
+
Yields:
|
| 152 |
+
A (context, page) tuple. Both are closed automatically.
|
| 153 |
+
"""
|
| 154 |
+
async with self._semaphore:
|
| 155 |
+
browser = await self._ensure_browser()
|
| 156 |
+
context: BrowserContext | None = None
|
| 157 |
+
try:
|
| 158 |
+
context = await browser.new_context(
|
| 159 |
+
viewport={
|
| 160 |
+
"width": viewport_width or settings.viewport_width,
|
| 161 |
+
"height": viewport_height or settings.viewport_height,
|
| 162 |
+
},
|
| 163 |
+
user_agent=None, # use Playwright default
|
| 164 |
+
)
|
| 165 |
+
page = await context.new_page()
|
| 166 |
+
yield context, page
|
| 167 |
+
finally:
|
| 168 |
+
if context:
|
| 169 |
+
await context.close()
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
browser_manager = BrowserManager()
|
|
@@ -0,0 +1,314 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CDP WebSocket proxy — expose browser CDP over a single port.
|
| 2 |
+
|
| 3 |
+
Handles WebSocket upgrade requests to ``/cdp`` and proxies CDP messages
|
| 4 |
+
between the external client and an internal Chromium CDP endpoint.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import asyncio
|
| 10 |
+
import base64
|
| 11 |
+
import hashlib
|
| 12 |
+
import logging
|
| 13 |
+
import os
|
| 14 |
+
import struct
|
| 15 |
+
from collections.abc import Callable, Coroutine
|
| 16 |
+
from typing import Any
|
| 17 |
+
|
| 18 |
+
from veilrender.auth import verify_token
|
| 19 |
+
from veilrender._vendor.httpserver import Request
|
| 20 |
+
|
| 21 |
+
logger = logging.getLogger(__name__)
|
| 22 |
+
|
| 23 |
+
# WebSocket opcodes
|
| 24 |
+
_OP_TEXT = 0x1
|
| 25 |
+
_OP_BINARY = 0x2
|
| 26 |
+
_OP_CLOSE = 0x8
|
| 27 |
+
_OP_PING = 0x9
|
| 28 |
+
_OP_PONG = 0xA
|
| 29 |
+
|
| 30 |
+
_WS_MAGIC = b"258EAFA5-E914-47DA-95CA-5B0D11CF9245"
|
| 31 |
+
|
| 32 |
+
# Maximum allowed WebSocket frame payload size (16 MB).
|
| 33 |
+
# Frames exceeding this limit are rejected to prevent OOM from
|
| 34 |
+
# malicious headers claiming excessively large payloads.
|
| 35 |
+
MAX_FRAME_SIZE = 16 * 1024 * 1024
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _accept_key(key: str) -> str:
|
| 39 |
+
"""Compute Sec-WebSocket-Accept from Sec-WebSocket-Key."""
|
| 40 |
+
digest = hashlib.sha1(key.encode() + _WS_MAGIC).digest()
|
| 41 |
+
return base64.b64encode(digest).decode()
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
async def _read_ws_frame(
|
| 45 |
+
reader: asyncio.StreamReader,
|
| 46 |
+
) -> tuple[bool, int, bytes] | None:
|
| 47 |
+
"""Read one WebSocket frame. Returns (fin, opcode, payload) or None on EOF."""
|
| 48 |
+
try:
|
| 49 |
+
head = await reader.readexactly(2)
|
| 50 |
+
except (asyncio.IncompleteReadError, ConnectionError):
|
| 51 |
+
return None
|
| 52 |
+
|
| 53 |
+
fin = bool(head[0] & 0x80)
|
| 54 |
+
opcode = head[0] & 0x0F
|
| 55 |
+
masked = bool(head[1] & 0x80)
|
| 56 |
+
length = head[1] & 0x7F
|
| 57 |
+
|
| 58 |
+
if length == 126:
|
| 59 |
+
raw = await reader.readexactly(2)
|
| 60 |
+
length = struct.unpack("!H", raw)[0]
|
| 61 |
+
elif length == 127:
|
| 62 |
+
raw = await reader.readexactly(8)
|
| 63 |
+
length = struct.unpack("!Q", raw)[0]
|
| 64 |
+
|
| 65 |
+
if length > MAX_FRAME_SIZE:
|
| 66 |
+
logger.warning("WebSocket frame too large (%d bytes), dropping", length)
|
| 67 |
+
return None
|
| 68 |
+
|
| 69 |
+
if masked:
|
| 70 |
+
mask = await reader.readexactly(4)
|
| 71 |
+
data = bytearray(await reader.readexactly(length))
|
| 72 |
+
for i in range(length):
|
| 73 |
+
data[i] ^= mask[i % 4]
|
| 74 |
+
return fin, opcode, bytes(data)
|
| 75 |
+
else:
|
| 76 |
+
data = await reader.readexactly(length)
|
| 77 |
+
return fin, opcode, data
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _make_ws_frame(
|
| 81 |
+
opcode: int, payload: bytes, *, fin: bool = True, mask: bool = False
|
| 82 |
+
) -> bytes:
|
| 83 |
+
"""Build a WebSocket frame."""
|
| 84 |
+
frame = bytearray()
|
| 85 |
+
frame.append((0x80 if fin else 0x00) | opcode)
|
| 86 |
+
|
| 87 |
+
length = len(payload)
|
| 88 |
+
if length < 126:
|
| 89 |
+
frame.append((0x80 if mask else 0) | length)
|
| 90 |
+
elif length < 65536:
|
| 91 |
+
frame.append((0x80 if mask else 0) | 126)
|
| 92 |
+
frame.extend(struct.pack("!H", length))
|
| 93 |
+
else:
|
| 94 |
+
frame.append((0x80 if mask else 0) | 127)
|
| 95 |
+
frame.extend(struct.pack("!Q", length))
|
| 96 |
+
|
| 97 |
+
if mask:
|
| 98 |
+
mask_key = os.urandom(4)
|
| 99 |
+
frame.extend(mask_key)
|
| 100 |
+
masked = bytearray(payload)
|
| 101 |
+
for i in range(length):
|
| 102 |
+
masked[i] ^= mask_key[i % 4]
|
| 103 |
+
frame.extend(masked)
|
| 104 |
+
else:
|
| 105 |
+
frame.extend(payload)
|
| 106 |
+
|
| 107 |
+
return bytes(frame)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
async def _proxy_ws(
|
| 111 |
+
src_reader: asyncio.StreamReader,
|
| 112 |
+
dst_writer: asyncio.StreamWriter,
|
| 113 |
+
*,
|
| 114 |
+
src_masked: bool,
|
| 115 |
+
dst_mask: bool,
|
| 116 |
+
label: str,
|
| 117 |
+
) -> None:
|
| 118 |
+
"""Forward WebSocket frames from src to dst until close/EOF."""
|
| 119 |
+
while True:
|
| 120 |
+
result = await _read_ws_frame(src_reader)
|
| 121 |
+
if result is None:
|
| 122 |
+
break
|
| 123 |
+
|
| 124 |
+
fin, opcode, payload = result
|
| 125 |
+
|
| 126 |
+
if opcode == _OP_CLOSE:
|
| 127 |
+
frame = _make_ws_frame(_OP_CLOSE, payload, fin=True, mask=dst_mask)
|
| 128 |
+
dst_writer.write(frame)
|
| 129 |
+
await dst_writer.drain()
|
| 130 |
+
break
|
| 131 |
+
elif opcode == _OP_PING:
|
| 132 |
+
# Forward pings as-is to the other side
|
| 133 |
+
frame = _make_ws_frame(_OP_PING, payload, fin=True, mask=dst_mask)
|
| 134 |
+
dst_writer.write(frame)
|
| 135 |
+
await dst_writer.drain()
|
| 136 |
+
else:
|
| 137 |
+
frame = _make_ws_frame(opcode, payload, fin=fin, mask=dst_mask)
|
| 138 |
+
dst_writer.write(frame)
|
| 139 |
+
await dst_writer.drain()
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def is_websocket_upgrade(method: str, headers: dict[str, str], path: str) -> bool:
|
| 143 |
+
"""Check if this HTTP request is a WebSocket upgrade to /cdp."""
|
| 144 |
+
if not path.startswith("/cdp"):
|
| 145 |
+
return False
|
| 146 |
+
upgrade = headers.get("upgrade", "").lower()
|
| 147 |
+
connection = headers.get("connection", "").lower()
|
| 148 |
+
return method == "GET" and "websocket" in upgrade and "upgrade" in connection
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
async def handle_cdp_upgrade(
|
| 152 |
+
reader: asyncio.StreamReader,
|
| 153 |
+
writer: asyncio.StreamWriter,
|
| 154 |
+
headers: dict[str, str],
|
| 155 |
+
path: str,
|
| 156 |
+
query_string: str,
|
| 157 |
+
get_cdp_url: Callable[[], Coroutine[Any, Any, str | None]],
|
| 158 |
+
) -> None:
|
| 159 |
+
"""Handle a WebSocket upgrade for CDP proxying.
|
| 160 |
+
|
| 161 |
+
Args:
|
| 162 |
+
reader: Client's stream reader.
|
| 163 |
+
writer: Client's stream writer.
|
| 164 |
+
headers: Parsed HTTP headers from the upgrade request.
|
| 165 |
+
path: Request path.
|
| 166 |
+
query_string: Raw query string.
|
| 167 |
+
get_cdp_url: Async callable that returns the internal CDP WebSocket URL,
|
| 168 |
+
or None if no browser is available.
|
| 169 |
+
"""
|
| 170 |
+
# Auth check — build a minimal Request object for verify_token
|
| 171 |
+
try:
|
| 172 |
+
req = Request(
|
| 173 |
+
method="GET",
|
| 174 |
+
path=path,
|
| 175 |
+
query_string=query_string,
|
| 176 |
+
headers=headers,
|
| 177 |
+
body=b"",
|
| 178 |
+
client_addr=writer.get_extra_info("peername") or ("0.0.0.0", 0),
|
| 179 |
+
)
|
| 180 |
+
verify_token(req)
|
| 181 |
+
except Exception:
|
| 182 |
+
writer.write(b"HTTP/1.1 401 Unauthorized\r\n\r\n")
|
| 183 |
+
await writer.drain()
|
| 184 |
+
writer.close()
|
| 185 |
+
return
|
| 186 |
+
|
| 187 |
+
# Get WebSocket accept key
|
| 188 |
+
ws_key = headers.get("sec-websocket-key", "")
|
| 189 |
+
if not ws_key:
|
| 190 |
+
writer.write(b"HTTP/1.1 400 Bad Request\r\n\r\nMissing Sec-WebSocket-Key\r\n")
|
| 191 |
+
await writer.drain()
|
| 192 |
+
writer.close()
|
| 193 |
+
return
|
| 194 |
+
|
| 195 |
+
# Get internal CDP URL
|
| 196 |
+
cdp_url = await get_cdp_url()
|
| 197 |
+
if cdp_url is None:
|
| 198 |
+
writer.write(
|
| 199 |
+
b"HTTP/1.1 503 Service Unavailable\r\n\r\nNo browser available\r\n"
|
| 200 |
+
)
|
| 201 |
+
await writer.drain()
|
| 202 |
+
writer.close()
|
| 203 |
+
return
|
| 204 |
+
|
| 205 |
+
# Connect to internal CDP
|
| 206 |
+
# Parse ws://host:port/path from cdp_url
|
| 207 |
+
from urllib.parse import urlparse
|
| 208 |
+
|
| 209 |
+
parsed = urlparse(cdp_url)
|
| 210 |
+
cdp_host = parsed.hostname or "127.0.0.1"
|
| 211 |
+
cdp_port = parsed.port or 9222
|
| 212 |
+
cdp_path = parsed.path or "/"
|
| 213 |
+
|
| 214 |
+
try:
|
| 215 |
+
cdp_reader, cdp_writer = await asyncio.open_connection(cdp_host, cdp_port)
|
| 216 |
+
except Exception as exc:
|
| 217 |
+
logger.error("Failed to connect to CDP at %s: %s", cdp_url, exc)
|
| 218 |
+
writer.write(b"HTTP/1.1 502 Bad Gateway\r\n\r\n")
|
| 219 |
+
await writer.drain()
|
| 220 |
+
writer.close()
|
| 221 |
+
return
|
| 222 |
+
|
| 223 |
+
# WebSocket handshake with internal CDP
|
| 224 |
+
cdp_ws_key = base64.b64encode(os.urandom(16)).decode()
|
| 225 |
+
ws_protocol = headers.get("sec-websocket-protocol", "")
|
| 226 |
+
handshake = (
|
| 227 |
+
f"GET {cdp_path} HTTP/1.1\r\n"
|
| 228 |
+
f"Host: {cdp_host}:{cdp_port}\r\n"
|
| 229 |
+
f"Upgrade: websocket\r\n"
|
| 230 |
+
f"Connection: Upgrade\r\n"
|
| 231 |
+
f"Sec-WebSocket-Version: 13\r\n"
|
| 232 |
+
f"Sec-WebSocket-Key: {cdp_ws_key}\r\n"
|
| 233 |
+
)
|
| 234 |
+
if ws_protocol:
|
| 235 |
+
handshake += f"Sec-WebSocket-Protocol: {ws_protocol}\r\n"
|
| 236 |
+
handshake += "\r\n"
|
| 237 |
+
cdp_writer.write(handshake.encode())
|
| 238 |
+
await cdp_writer.drain()
|
| 239 |
+
|
| 240 |
+
# Read CDP handshake response
|
| 241 |
+
cdp_response = b""
|
| 242 |
+
while b"\r\n\r\n" not in cdp_response:
|
| 243 |
+
chunk = await cdp_reader.read(4096)
|
| 244 |
+
if not chunk:
|
| 245 |
+
logger.error("CDP handshake failed: connection closed")
|
| 246 |
+
writer.write(b"HTTP/1.1 502 Bad Gateway\r\n\r\n")
|
| 247 |
+
await writer.drain()
|
| 248 |
+
writer.close()
|
| 249 |
+
cdp_writer.close()
|
| 250 |
+
return
|
| 251 |
+
cdp_response += chunk
|
| 252 |
+
|
| 253 |
+
if b"101" not in cdp_response.split(b"\r\n")[0]:
|
| 254 |
+
logger.error("CDP handshake failed: %s", cdp_response[:200])
|
| 255 |
+
writer.write(b"HTTP/1.1 502 Bad Gateway\r\n\r\n")
|
| 256 |
+
await writer.drain()
|
| 257 |
+
writer.close()
|
| 258 |
+
cdp_writer.close()
|
| 259 |
+
return
|
| 260 |
+
|
| 261 |
+
# Complete WebSocket handshake with the external client
|
| 262 |
+
accept = _accept_key(ws_key)
|
| 263 |
+
response = (
|
| 264 |
+
f"HTTP/1.1 101 Switching Protocols\r\n"
|
| 265 |
+
f"Upgrade: websocket\r\n"
|
| 266 |
+
f"Connection: Upgrade\r\n"
|
| 267 |
+
f"Sec-WebSocket-Accept: {accept}\r\n"
|
| 268 |
+
)
|
| 269 |
+
if ws_protocol:
|
| 270 |
+
response += f"Sec-WebSocket-Protocol: {ws_protocol}\r\n"
|
| 271 |
+
response += "\r\n"
|
| 272 |
+
writer.write(response.encode())
|
| 273 |
+
await writer.drain()
|
| 274 |
+
|
| 275 |
+
logger.info("CDP proxy session started")
|
| 276 |
+
|
| 277 |
+
# Bidirectional proxy — when one direction ends, cancel the other
|
| 278 |
+
t1 = asyncio.create_task(
|
| 279 |
+
_proxy_ws(
|
| 280 |
+
reader,
|
| 281 |
+
cdp_writer,
|
| 282 |
+
src_masked=True,
|
| 283 |
+
dst_mask=True,
|
| 284 |
+
label="client→cdp",
|
| 285 |
+
)
|
| 286 |
+
)
|
| 287 |
+
t2 = asyncio.create_task(
|
| 288 |
+
_proxy_ws(
|
| 289 |
+
cdp_reader,
|
| 290 |
+
writer,
|
| 291 |
+
src_masked=False,
|
| 292 |
+
dst_mask=False,
|
| 293 |
+
label="cdp→client",
|
| 294 |
+
)
|
| 295 |
+
)
|
| 296 |
+
|
| 297 |
+
try:
|
| 298 |
+
done, pending = await asyncio.wait(
|
| 299 |
+
{t1, t2}, return_when=asyncio.FIRST_COMPLETED
|
| 300 |
+
)
|
| 301 |
+
for t in pending:
|
| 302 |
+
t.cancel()
|
| 303 |
+
# Await cancelled tasks to suppress warnings
|
| 304 |
+
for t in pending:
|
| 305 |
+
try:
|
| 306 |
+
await t
|
| 307 |
+
except asyncio.CancelledError:
|
| 308 |
+
pass
|
| 309 |
+
except Exception as exc:
|
| 310 |
+
logger.debug("CDP proxy ended: %s", exc)
|
| 311 |
+
finally:
|
| 312 |
+
logger.info("CDP proxy session ended")
|
| 313 |
+
cdp_writer.close()
|
| 314 |
+
writer.close()
|
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Application settings loaded from environment variables."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class Settings:
|
| 9 |
+
"""Configuration from ``VEILRENDER_*`` environment variables."""
|
| 10 |
+
|
| 11 |
+
def __init__(self) -> None:
|
| 12 |
+
self.api_token: str | None = os.environ.get("VEILRENDER_API_TOKEN")
|
| 13 |
+
self.port: int = int(os.environ.get("VEILRENDER_PORT", "7860"))
|
| 14 |
+
self.host: str = os.environ.get("VEILRENDER_HOST", "0.0.0.0")
|
| 15 |
+
self.timeout: int = int(os.environ.get("VEILRENDER_TIMEOUT", "30000"))
|
| 16 |
+
self.viewport_width: int = int(
|
| 17 |
+
os.environ.get("VEILRENDER_VIEWPORT_WIDTH", "1280")
|
| 18 |
+
)
|
| 19 |
+
self.viewport_height: int = int(
|
| 20 |
+
os.environ.get("VEILRENDER_VIEWPORT_HEIGHT", "720")
|
| 21 |
+
)
|
| 22 |
+
self.max_concurrent: int = int(os.environ.get("VEILRENDER_MAX_CONCURRENT", "3"))
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
settings = Settings()
|
|
@@ -0,0 +1,109 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Request and response data models."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass, field
|
| 6 |
+
from typing import Any, Literal
|
| 7 |
+
|
| 8 |
+
WaitUntil = Literal["commit", "domcontentloaded", "load", "networkidle"]
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class RenderRequest:
|
| 13 |
+
"""POST /render request body."""
|
| 14 |
+
|
| 15 |
+
url: str
|
| 16 |
+
formats: list[str] = field(
|
| 17 |
+
default_factory=lambda: ["html", "markdown", "readability"]
|
| 18 |
+
)
|
| 19 |
+
wait_until: WaitUntil = "networkidle"
|
| 20 |
+
timeout: int | None = None # ms, None = use default
|
| 21 |
+
|
| 22 |
+
@classmethod
|
| 23 |
+
def from_dict(cls, data: dict[str, Any]) -> RenderRequest:
|
| 24 |
+
"""Create from parsed JSON dict."""
|
| 25 |
+
return cls(
|
| 26 |
+
url=data["url"],
|
| 27 |
+
formats=data.get("formats", ["html", "markdown", "readability"]),
|
| 28 |
+
wait_until=data.get("wait_until", "networkidle"),
|
| 29 |
+
timeout=data.get("timeout"),
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
@dataclass
|
| 34 |
+
class ScreenshotRequest:
|
| 35 |
+
"""POST /screenshot request body."""
|
| 36 |
+
|
| 37 |
+
url: str
|
| 38 |
+
full_page: bool = False
|
| 39 |
+
wait_until: WaitUntil = "networkidle"
|
| 40 |
+
timeout: int | None = None
|
| 41 |
+
viewport_width: int | None = None
|
| 42 |
+
viewport_height: int | None = None
|
| 43 |
+
|
| 44 |
+
@classmethod
|
| 45 |
+
def from_dict(cls, data: dict[str, Any]) -> ScreenshotRequest:
|
| 46 |
+
"""Create from parsed JSON dict."""
|
| 47 |
+
return cls(
|
| 48 |
+
url=data["url"],
|
| 49 |
+
full_page=data.get("full_page", False),
|
| 50 |
+
wait_until=data.get("wait_until", "networkidle"),
|
| 51 |
+
timeout=data.get("timeout"),
|
| 52 |
+
viewport_width=data.get("viewport_width"),
|
| 53 |
+
viewport_height=data.get("viewport_height"),
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
@dataclass
|
| 58 |
+
class LinkInfo:
|
| 59 |
+
"""Extracted link from a page."""
|
| 60 |
+
|
| 61 |
+
url: str
|
| 62 |
+
text: str
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
@dataclass
|
| 66 |
+
class PageMetadata:
|
| 67 |
+
"""Metadata extracted from a rendered page."""
|
| 68 |
+
|
| 69 |
+
title: str
|
| 70 |
+
url: str
|
| 71 |
+
status_code: int
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
@dataclass
|
| 75 |
+
class RenderContent:
|
| 76 |
+
"""Rendered content in multiple formats."""
|
| 77 |
+
|
| 78 |
+
html: str | None = None
|
| 79 |
+
markdown: str | None = None
|
| 80 |
+
readability: str | None = None
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
@dataclass
|
| 84 |
+
class RenderResponse:
|
| 85 |
+
"""POST /render response body."""
|
| 86 |
+
|
| 87 |
+
content: RenderContent
|
| 88 |
+
metadata: PageMetadata
|
| 89 |
+
links: list[LinkInfo]
|
| 90 |
+
|
| 91 |
+
def to_dict(self) -> dict[str, Any]:
|
| 92 |
+
"""Serialize to JSON-compatible dict."""
|
| 93 |
+
return {
|
| 94 |
+
"content": {
|
| 95 |
+
k: v
|
| 96 |
+
for k, v in {
|
| 97 |
+
"html": self.content.html,
|
| 98 |
+
"markdown": self.content.markdown,
|
| 99 |
+
"readability": self.content.readability,
|
| 100 |
+
}.items()
|
| 101 |
+
if v is not None
|
| 102 |
+
},
|
| 103 |
+
"metadata": {
|
| 104 |
+
"title": self.metadata.title,
|
| 105 |
+
"url": self.metadata.url,
|
| 106 |
+
"status_code": self.metadata.status_code,
|
| 107 |
+
},
|
| 108 |
+
"links": [{"url": link.url, "text": link.text} for link in self.links],
|
| 109 |
+
}
|
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Route handlers."""
|
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Health check endpoint."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from veilrender._vendor.httpserver import App, JSONResponse, Request
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def register(app: App) -> None:
|
| 9 |
+
"""Register health check routes on the app."""
|
| 10 |
+
|
| 11 |
+
@app.get("/health")
|
| 12 |
+
async def health(request: Request) -> JSONResponse:
|
| 13 |
+
return JSONResponse({"status": "ok"})
|
| 14 |
+
|
| 15 |
+
@app.get("/")
|
| 16 |
+
async def root(request: Request) -> JSONResponse:
|
| 17 |
+
return JSONResponse({"service": "veilrender", "status": "ok"})
|
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Page rendering endpoint."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
import re
|
| 7 |
+
|
| 8 |
+
from veilrender._vendor.httpserver import App, JSONResponse, Request
|
| 9 |
+
from veilrender._vendor.readability import extract as readability_extract
|
| 10 |
+
from veilrender._vendor.soup import Soup
|
| 11 |
+
from veilrender.auth import verify_token
|
| 12 |
+
from veilrender.browser import browser_manager
|
| 13 |
+
from veilrender.config import settings
|
| 14 |
+
from veilrender.models import (
|
| 15 |
+
LinkInfo,
|
| 16 |
+
PageMetadata,
|
| 17 |
+
RenderContent,
|
| 18 |
+
RenderRequest,
|
| 19 |
+
RenderResponse,
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
logger = logging.getLogger(__name__)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _extract_links(html: str, base_url: str) -> list[LinkInfo]:
|
| 26 |
+
"""Extract links from HTML using zerodep soup."""
|
| 27 |
+
links: list[LinkInfo] = []
|
| 28 |
+
try:
|
| 29 |
+
soup = Soup(html)
|
| 30 |
+
for a in soup.find_all("a", href=True):
|
| 31 |
+
href = a.get("href", "")
|
| 32 |
+
text = a.get_text(strip=True)
|
| 33 |
+
if href and not href.startswith(("#", "javascript:")):
|
| 34 |
+
links.append(LinkInfo(url=href, text=text))
|
| 35 |
+
except Exception:
|
| 36 |
+
logger.debug("Failed to extract links", exc_info=True)
|
| 37 |
+
return links
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _html_to_markdown(html: str) -> str:
|
| 41 |
+
"""Simple HTML to Markdown conversion using readability text output.
|
| 42 |
+
|
| 43 |
+
This is a lightweight conversion: we extract readable content and
|
| 44 |
+
return the plain-text representation, which is sufficient for LLM
|
| 45 |
+
consumption. For full-fidelity HTML→Markdown, a dedicated converter
|
| 46 |
+
(like markdownify) would be needed.
|
| 47 |
+
"""
|
| 48 |
+
try:
|
| 49 |
+
result = readability_extract(html)
|
| 50 |
+
return result.text
|
| 51 |
+
except Exception:
|
| 52 |
+
# Fallback: strip tags and return raw text
|
| 53 |
+
return re.sub(r"<[^>]+>", "", html)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def register(app: App) -> None:
|
| 57 |
+
"""Register render routes on the app."""
|
| 58 |
+
|
| 59 |
+
@app.post("/render")
|
| 60 |
+
async def render(request: Request) -> JSONResponse:
|
| 61 |
+
verify_token(request)
|
| 62 |
+
|
| 63 |
+
try:
|
| 64 |
+
data = request.json()
|
| 65 |
+
except Exception:
|
| 66 |
+
return JSONResponse({"error": "Invalid JSON body"}, status_code=400)
|
| 67 |
+
|
| 68 |
+
if "url" not in data:
|
| 69 |
+
return JSONResponse({"error": "Missing 'url' field"}, status_code=400)
|
| 70 |
+
|
| 71 |
+
req = RenderRequest.from_dict(data)
|
| 72 |
+
timeout = req.timeout or settings.timeout
|
| 73 |
+
|
| 74 |
+
try:
|
| 75 |
+
async with browser_manager.get_page() as (ctx, page):
|
| 76 |
+
response = await page.goto(
|
| 77 |
+
req.url,
|
| 78 |
+
wait_until=req.wait_until,
|
| 79 |
+
timeout=timeout,
|
| 80 |
+
)
|
| 81 |
+
status_code = response.status if response else 0
|
| 82 |
+
title = await page.title()
|
| 83 |
+
final_url = page.url
|
| 84 |
+
html = await page.content()
|
| 85 |
+
except Exception as exc:
|
| 86 |
+
logger.error("Render failed for %s: %s", req.url, exc)
|
| 87 |
+
return JSONResponse(
|
| 88 |
+
{"error": f"Render failed: {exc!s}"},
|
| 89 |
+
status_code=502,
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
# Build content based on requested formats
|
| 93 |
+
content = RenderContent()
|
| 94 |
+
|
| 95 |
+
if "html" in req.formats:
|
| 96 |
+
content.html = html
|
| 97 |
+
|
| 98 |
+
if "readability" in req.formats:
|
| 99 |
+
try:
|
| 100 |
+
rd = readability_extract(html, url=final_url)
|
| 101 |
+
content.readability = rd.text
|
| 102 |
+
except Exception:
|
| 103 |
+
logger.debug("Readability extraction failed", exc_info=True)
|
| 104 |
+
content.readability = None
|
| 105 |
+
|
| 106 |
+
if "markdown" in req.formats:
|
| 107 |
+
content.markdown = _html_to_markdown(html)
|
| 108 |
+
|
| 109 |
+
# Extract links
|
| 110 |
+
links = _extract_links(html, final_url)
|
| 111 |
+
|
| 112 |
+
result = RenderResponse(
|
| 113 |
+
content=content,
|
| 114 |
+
metadata=PageMetadata(
|
| 115 |
+
title=title,
|
| 116 |
+
url=final_url,
|
| 117 |
+
status_code=status_code,
|
| 118 |
+
),
|
| 119 |
+
links=links,
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
return JSONResponse(result.to_dict())
|
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Screenshot endpoint."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
|
| 7 |
+
from veilrender._vendor.httpserver import App, Request, Response
|
| 8 |
+
from veilrender.auth import verify_token
|
| 9 |
+
from veilrender.browser import browser_manager
|
| 10 |
+
from veilrender.config import settings
|
| 11 |
+
from veilrender.models import ScreenshotRequest
|
| 12 |
+
|
| 13 |
+
logger = logging.getLogger(__name__)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def register(app: App) -> None:
|
| 17 |
+
"""Register screenshot routes on the app."""
|
| 18 |
+
|
| 19 |
+
@app.post("/screenshot")
|
| 20 |
+
async def screenshot(request: Request) -> Response:
|
| 21 |
+
verify_token(request)
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
data = request.json()
|
| 25 |
+
except Exception:
|
| 26 |
+
return Response(
|
| 27 |
+
body=b'{"error": "Invalid JSON body"}',
|
| 28 |
+
status_code=400,
|
| 29 |
+
content_type="application/json",
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
if "url" not in data:
|
| 33 |
+
return Response(
|
| 34 |
+
body=b'{"error": "Missing \'url\' field"}',
|
| 35 |
+
status_code=400,
|
| 36 |
+
content_type="application/json",
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
req = ScreenshotRequest.from_dict(data)
|
| 40 |
+
timeout = req.timeout or settings.timeout
|
| 41 |
+
|
| 42 |
+
try:
|
| 43 |
+
async with browser_manager.get_page(
|
| 44 |
+
viewport_width=req.viewport_width,
|
| 45 |
+
viewport_height=req.viewport_height,
|
| 46 |
+
) as (ctx, page):
|
| 47 |
+
await page.goto(
|
| 48 |
+
req.url,
|
| 49 |
+
wait_until=req.wait_until,
|
| 50 |
+
timeout=timeout,
|
| 51 |
+
)
|
| 52 |
+
png_bytes = await page.screenshot(full_page=req.full_page)
|
| 53 |
+
except Exception as exc:
|
| 54 |
+
logger.error("Screenshot failed for %s: %s", req.url, exc)
|
| 55 |
+
return Response(
|
| 56 |
+
body=f'{{"error": "Screenshot failed: {exc!s}"}}'.encode(),
|
| 57 |
+
status_code=502,
|
| 58 |
+
content_type="application/json",
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
return Response(
|
| 62 |
+
body=png_bytes,
|
| 63 |
+
status_code=200,
|
| 64 |
+
content_type="image/png",
|
| 65 |
+
)
|