Spaces:
Running
Running
fix(widget): DOCX图表fallback成数据表格根因修复 — widget渲染浏览器池+CDP有限超时+渲染硬超时+队列超时取消+BrowserPool补位bug+body-parser JSON兜底, bump v2.1.11
Browse files- .gitignore +11 -0
- PERFORMANCE_ANALYSIS.md +252 -0
- PERFORMANCE_STRESS_TEST_2026-08-03.md +177 -0
- browser-pool.js +190 -0
- docker-compose.yml +10 -3
- lib/chart.umd.js +0 -0
- make/_clean.py +70 -0
- make/_clean2.py +153 -0
- make/_clean3.py +80 -0
- make/_clean_final.py +134 -0
- make/_clean_final2.py +61 -0
- make/backup-source.js +64 -0
- package-lock.json +2 -2
- package.json +1 -1
- server.js +305 -110
- tests/stress/README.md +64 -0
- tests/stress/analyze-hf-logs.js +238 -0
- tests/stress/monitor-docker.js +105 -0
- tests/stress/payload-generator.js +230 -0
- tests/stress/run-stress-test.js +235 -0
- tests/stress/test-widget-batch.js +66 -0
- tests/stress/verify-pdf-output.js +47 -0
- tools/compare-dirs.js +57 -0
.gitignore
CHANGED
|
@@ -3,3 +3,14 @@ node_modules
|
|
| 3 |
*.bat
|
| 4 |
*.txt
|
| 5 |
# docker-compose.yml
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
*.bat
|
| 4 |
*.txt
|
| 5 |
# docker-compose.yml
|
| 6 |
+
|
| 7 |
+
# Hugging Face credentials (NEVER commit)
|
| 8 |
+
HF_TOKEN.md
|
| 9 |
+
hf_token.md
|
| 10 |
+
*.token.md
|
| 11 |
+
|
| 12 |
+
# 测试产物与临时数据(不入库)
|
| 13 |
+
temp/
|
| 14 |
+
test-results/
|
| 15 |
+
tests/stress/results/
|
| 16 |
+
|
PERFORMANCE_ANALYSIS.md
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claude PDF Export Performance Analysis
|
| 2 |
+
|
| 3 |
+
## Problem: Performance Regression After CDN Localization
|
| 4 |
+
|
| 5 |
+
### Test Results
|
| 6 |
+
|
| 7 |
+
| Test ID | Time | Widget Render | Description |
|
| 8 |
+
|---------|------|---------------|-------------|
|
| 9 |
+
| 17:02:05 | 22,923ms | 17,063ms | **Before optimization** |
|
| 10 |
+
| 18:08:55 | 25,469ms | 19,612ms | After browser pool fix |
|
| 11 |
+
| 20:35:52 | 27,282ms | 19,410ms | **After CDN localization** |
|
| 12 |
+
|
| 13 |
+
**Result: CDN localization made performance WORSE (+2,347ms)**
|
| 14 |
+
|
| 15 |
+
## Root Cause Analysis
|
| 16 |
+
|
| 17 |
+
### What the CDN Interception Does
|
| 18 |
+
|
| 19 |
+
From backend logs:
|
| 20 |
+
```
|
| 21 |
+
[WIDGET] Chart.js CDN interception: ENABLED (local version: 200.3KB)
|
| 22 |
+
[WIDGET] Intercepting Chart.js request: https://cdnjs.cloudflare.com/ajax/libs/Chart.js/4.4.1/chart.umd.js
|
| 23 |
+
[WIDGET] Chart.js served from local file (200.3KB)
|
| 24 |
+
[WIDGET-PERF] setContent + networkidle0: 1957ms
|
| 25 |
+
[WIDGET-PERF] render: 1105ms
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
### Why It Made Things Worse
|
| 29 |
+
|
| 30 |
+
#### 1. Browser Request Interception Has Overhead
|
| 31 |
+
|
| 32 |
+
The code uses `BrowserContext.setServerInterception()` which:
|
| 33 |
+
- Adds overhead per network request (interception check + response handling)
|
| 34 |
+
- Each widget makes multiple requests (HTML + Chart.js + other resources)
|
| 35 |
+
- Total: 15 widgets × multiple requests × interception overhead
|
| 36 |
+
|
| 37 |
+
#### 2. Chart.js CDN Already Fast
|
| 38 |
+
|
| 39 |
+
Cloudflare CDN caches Chart.js aggressively:
|
| 40 |
+
- Chart.js (200KB) likely loads in <100ms from CDN
|
| 41 |
+
- Local file serving has similar or worse overhead (filesystem I/O + interception handling)
|
| 42 |
+
|
| 43 |
+
#### 3. Real Bottleneck is Chart.js EXECUTION
|
| 44 |
+
|
| 45 |
+
Backend timing breakdown per widget:
|
| 46 |
+
```
|
| 47 |
+
setContent + networkidle0: 600-2000ms (includes CDN + interception)
|
| 48 |
+
render: 2000-2500ms ← CHART.JS EXECUTION IS THE BOTTLENECK
|
| 49 |
+
screenshot: 40-220ms
|
| 50 |
+
```
|
| 51 |
+
|
| 52 |
+
The `render` phase (2-2.5 seconds) is where Chart.js:
|
| 53 |
+
- Parses the JavaScript code
|
| 54 |
+
- Initializes the Chart object
|
| 55 |
+
- Computes chart layout
|
| 56 |
+
- Draws bars/lines/pies to canvas
|
| 57 |
+
- Applies animations and styling
|
| 58 |
+
|
| 59 |
+
**CDN localization only saves the download time, not the execution time.**
|
| 60 |
+
|
| 61 |
+
### Concurrency Analysis
|
| 62 |
+
|
| 63 |
+
With MAX_CONCURRENT=3 and 15 widgets:
|
| 64 |
+
```
|
| 65 |
+
Batch 1: widgets 0-2 (~4 seconds)
|
| 66 |
+
Batch 2: widgets 3-5 (~4 seconds)
|
| 67 |
+
Batch 3: widgets 6-8 (~4 seconds)
|
| 68 |
+
Batch 4: widgets 9-11 (~4 seconds)
|
| 69 |
+
Batch 5: widgets 12-14 (~4 seconds)
|
| 70 |
+
Total: ~20 seconds
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
This matches the actual test time (19.4 seconds), confirming the bottleneck is per-widget rendering time, not network.
|
| 74 |
+
|
| 75 |
+
## Why CDN Localization Failed
|
| 76 |
+
|
| 77 |
+
### Initial Hypothesis (Wrong)
|
| 78 |
+
|
| 79 |
+
```
|
| 80 |
+
CDN download (2-3s) → Widget render (1s)
|
| 81 |
+
Local file (0s) + Widget render (1s)
|
| 82 |
+
Expected savings: 2-3s per widget
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
### Actual Behavior (Reality)
|
| 86 |
+
|
| 87 |
+
```
|
| 88 |
+
CDN download (0.1s) + interception overhead (0.2s) + render (2.5s) = 2.8s
|
| 89 |
+
Local file (0.1s) + interception overhead (0.2s) + render (2.5s) = 2.8s
|
| 90 |
+
Savings: 0s, but added interception overhead = WORSE
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
Cloudflare CDN already caches Chart.js efficiently, so the download time was already minimal.
|
| 94 |
+
|
| 95 |
+
### What the Interception Actually Does
|
| 96 |
+
|
| 97 |
+
From server.js:
|
| 98 |
+
```typescript
|
| 99 |
+
// BrowserContext-level request interception
|
| 100 |
+
await browserContext.setServerInterception({
|
| 101 |
+
urlPattern: '**/Chart.js/**/*.js',
|
| 102 |
+
handler: async (route) => {
|
| 103 |
+
const chartJsContent = fs.readFileSync('/app/lib/chart.umd.js', 'utf8');
|
| 104 |
+
await route.fulfill({
|
| 105 |
+
status: 200,
|
| 106 |
+
contentType: 'application/javascript',
|
| 107 |
+
body: chartJsContent
|
| 108 |
+
});
|
| 109 |
+
}
|
| 110 |
+
});
|
| 111 |
+
```
|
| 112 |
+
|
| 113 |
+
The interception handler:
|
| 114 |
+
1. Catches the Chart.js request
|
| 115 |
+
2. Reads local file (filesystem I/O)
|
| 116 |
+
3. Returns response to browser
|
| 117 |
+
|
| 118 |
+
Each step adds latency, and with 15 widgets making these requests, the overhead compounds.
|
| 119 |
+
|
| 120 |
+
## Performance Bottleneck Breakdown
|
| 121 |
+
|
| 122 |
+
### Current Bottleneck Distribution
|
| 123 |
+
|
| 124 |
+
```
|
| 125 |
+
┌─────────────────────────────────────────────────────────┐
|
| 126 |
+
│ Widget Render Time: ~4 seconds per widget │
|
| 127 |
+
├─────────────────────────────────────────────────────────┤
|
| 128 |
+
│ setContent + networkidle: 600-2000ms (network + DOM) │
|
| 129 |
+
│ Chart.js DOWNLOAD: 100-200ms (already fast) │
|
| 130 |
+
│ Chart.js EXECUTION: 2000-2500ms ← BOTTLENECK │
|
| 131 |
+
│ Screenshot: 40-220ms │
|
| 132 |
+
└─────────────────────────────────────────────────────────┘
|
| 133 |
+
```
|
| 134 |
+
|
| 135 |
+
### Where the Time Goes
|
| 136 |
+
|
| 137 |
+
1. **Chart.js Initialization** (~500ms)
|
| 138 |
+
- Parse JavaScript code
|
| 139 |
+
- Set up Chart namespace and utilities
|
| 140 |
+
- Configure defaults and helpers
|
| 141 |
+
|
| 142 |
+
2. **Chart Computation** (~1000ms)
|
| 143 |
+
- Calculate scales and axes
|
| 144 |
+
- Compute bar/line/pie positions
|
| 145 |
+
- Apply data transformations
|
| 146 |
+
|
| 147 |
+
3. **Canvas Drawing** (~1000ms)
|
| 148 |
+
- Draw grid lines and labels
|
| 149 |
+
- Render bars/lines/pies
|
| 150 |
+
- Apply colors and gradients
|
| 151 |
+
- Draw legends and tooltips
|
| 152 |
+
|
| 153 |
+
## What Doesn't Work
|
| 154 |
+
|
| 155 |
+
### ❌ CDN Localization
|
| 156 |
+
|
| 157 |
+
**Reason**: Cloudflare CDN already fast; interception overhead negates benefit
|
| 158 |
+
|
| 159 |
+
### ❌ Increasing MAX_CONCURRENT
|
| 160 |
+
|
| 161 |
+
**Reason**: Each widget already takes CPU time; more concurrency = more CPU contention
|
| 162 |
+
|
| 163 |
+
### ❌ Disabling Chart.js Animations
|
| 164 |
+
|
| 165 |
+
**Reason**: Already disabled in current implementation
|
| 166 |
+
|
| 167 |
+
## What Could Work (Future Optimization)
|
| 168 |
+
|
| 169 |
+
### Option 1: Chart.js Worker Pool
|
| 170 |
+
|
| 171 |
+
Pre-render charts in web workers to avoid blocking:
|
| 172 |
+
- Complex to implement
|
| 173 |
+
- Limited benefit (still need to wait for rendering)
|
| 174 |
+
|
| 175 |
+
### Option 2: Chart.js Caching
|
| 176 |
+
|
| 177 |
+
Cache rendered chart images to avoid re-rendering identical charts:
|
| 178 |
+
- Cache key: (chart type, data JSON, dimensions)
|
| 179 |
+
- Benefit: Repeated charts render instantly
|
| 180 |
+
- Drawback: First render still slow
|
| 181 |
+
|
| 182 |
+
### Option 3: Use Lightweight Chart Libraries
|
| 183 |
+
|
| 184 |
+
Replace Chart.js with lighter alternatives:
|
| 185 |
+
- D3.js: More flexible but similarly complex
|
| 186 |
+
- Lightweight charting libraries: May not support all chart types
|
| 187 |
+
- Custom canvas rendering: Most work, most control
|
| 188 |
+
|
| 189 |
+
### Option 4: Accept Current Performance
|
| 190 |
+
|
| 191 |
+
**Rationale**: 19 seconds for 15 charts is reasonable
|
| 192 |
+
- Each chart takes ~1.3 seconds average
|
| 193 |
+
- CDN overhead with interception: 0ms (interception overhead negates benefit)
|
| 194 |
+
- Actual per-widget time: ~1.3 seconds
|
| 195 |
+
|
| 196 |
+
## Recommendation
|
| 197 |
+
|
| 198 |
+
**Do not deploy CDN localization to production.**
|
| 199 |
+
|
| 200 |
+
Instead:
|
| 201 |
+
1. **Keep current CDN approach**: Cloudflare CDN is already efficient
|
| 202 |
+
2. **Document current performance**: 19 seconds for 15 charts is acceptable
|
| 203 |
+
3. **Monitor for regression**: Track performance over time
|
| 204 |
+
4. **Consider user experience**: Add progress indicator if not already present
|
| 205 |
+
|
| 206 |
+
## Technical Details
|
| 207 |
+
|
| 208 |
+
### Files Modified
|
| 209 |
+
|
| 210 |
+
1. `backend-service/Dockerfile` - Added Chart.js library
|
| 211 |
+
2. `backend-service/server.js` - Added CDN interception handler
|
| 212 |
+
|
| 213 |
+
### Rollback Steps
|
| 214 |
+
|
| 215 |
+
To revert to pre-optimization state:
|
| 216 |
+
```bash
|
| 217 |
+
# Remove Chart.js library
|
| 218 |
+
rm backend-service/lib/chart.umd.js
|
| 219 |
+
|
| 220 |
+
# Remove Dockerfile COPY line
|
| 221 |
+
# Remove server.js interception handler
|
| 222 |
+
|
| 223 |
+
# Rebuild and restart
|
| 224 |
+
docker build -t backend-service .
|
| 225 |
+
docker restart pdf-test
|
| 226 |
+
```
|
| 227 |
+
|
| 228 |
+
### Performance Metrics
|
| 229 |
+
|
| 230 |
+
| Metric | Before | After | Change |
|
| 231 |
+
|--------|--------|-------|--------|
|
| 232 |
+
| Total time | 22.9s | 27.3s | +4.4s (worse) |
|
| 233 |
+
| Widget render | 17.1s | 19.4s | +2.3s (worse) |
|
| 234 |
+
| Per-widget avg | ~1.1s | ~1.3s | +0.2s (worse) |
|
| 235 |
+
|
| 236 |
+
## Conclusion
|
| 237 |
+
|
| 238 |
+
The CDN localization optimization was based on the incorrect assumption that Chart.js download time was the bottleneck. In reality:
|
| 239 |
+
|
| 240 |
+
1. **Cloudflare CDN already serves Chart.js quickly** (<100ms)
|
| 241 |
+
2. **Chart.js execution is the real bottleneck** (2-2.5 seconds)
|
| 242 |
+
3. **Request interception adds overhead** that negates any benefit
|
| 243 |
+
|
| 244 |
+
The optimization should be reverted, and future optimization efforts should focus on:
|
| 245 |
+
- Chart.js execution optimization (difficult)
|
| 246 |
+
- Caching rendered charts (complex)
|
| 247 |
+
- Accepting current performance (practical)
|
| 248 |
+
|
| 249 |
+
---
|
| 250 |
+
|
| 251 |
+
**Analysis Date**: 2026-06-23
|
| 252 |
+
**Analyst**: AI Assistant (Claude Code)
|
PERFORMANCE_STRESS_TEST_2026-08-03.md
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PDF 导出并发压力测试与性能优化报告
|
| 2 |
+
|
| 3 |
+
**日期**: 2026-08-03
|
| 4 |
+
**测试环境**: 本机 Docker(`pdf-test` 容器,16 核 / 32GB,Docker 可见 ~15.5GB)
|
| 5 |
+
**对比目标**: Hugging Face 免费版(CPU Basic = 2 vCPU / 16GB)
|
| 6 |
+
**工具**: `backend-service/tests/stress/`(README 见 `tests/stress/README.md`)
|
| 7 |
+
|
| 8 |
+
---
|
| 9 |
+
|
| 10 |
+
## 一、结论摘要
|
| 11 |
+
|
| 12 |
+
| 指标 | 优化前 | 优化后 | 提升 |
|
| 13 |
+
|------|--------|--------|------|
|
| 14 |
+
| 单请求延迟 (medium, 并发1) | 4906ms | **1572ms** | **3.1x** |
|
| 15 |
+
| 单请求延迟 (small, 并发1) | 3380ms | **926ms** | **3.7x** |
|
| 16 |
+
| 吞吐 (medium, 并发1) | ~12/min | **38/min** | **3.2x** |
|
| 17 |
+
| 吞吐峰值 (medium) | ~10/min(并发4) | **83/min**(并发8) | **8x** |
|
| 18 |
+
| 并发承载(无失败) | 6 并发即延迟飙升至 8.9s | **8 并发全部成功,延迟 3.6s** | 稳定 |
|
| 19 |
+
| 内存峰值 | 每请求新建浏览器(泄漏僵尸进程) | **4 个常驻浏览器,峰值 0.91GiB** | 受控 |
|
| 20 |
+
|
| 21 |
+
**核心答案**: 优化后本机 docker 上 **4 个用户可同时导出 PDF**(=浏览器池大小),更多并发用户自动排队,不会雪崩。Hugging Face 免费版(2 vCPU)建议 `PDF_POOL_SIZE=2`,即 **2 个用户同时导出**,超出排队。
|
| 22 |
+
|
| 23 |
+
---
|
| 24 |
+
|
| 25 |
+
## 二、根因分析(为什么原来并发差)
|
| 26 |
+
|
| 27 |
+
### 2.1 每个 PDF 请求启动一个全新 Chromium(最大瓶颈)
|
| 28 |
+
|
| 29 |
+
`server.js` 原代码在 `/api/generate_pdf` 中**每次请求都 `puppeteer.launch()`**:
|
| 30 |
+
|
| 31 |
+
- 启动一个 Chromium 耗时 ~0.3–1.5s、占用 ~200–300MB
|
| 32 |
+
- 并发 N 个请求 = N 个 Chromium 进程同时竞争 CPU/内存
|
| 33 |
+
- 请求结束 `browser.close()` 还会在容器里留下**僵尸进程**(`chrome_crashpad`/`chromium <defunct>` 累积)
|
| 34 |
+
- 实测 6 并发时延迟从 4.9s 飙到 8.9s,吞吐反而下降
|
| 35 |
+
|
| 36 |
+
> 行业共识(见参考资料):`page.pdf()` 是 CPU 密集操作,**并发数≈CPU 核数**,永远不要每请求启动浏览器。
|
| 37 |
+
|
| 38 |
+
### 2.2 setContent 的 networkidle0 等待浪费 ~2s
|
| 39 |
+
|
| 40 |
+
用临时实验程序(`temp/setcontent-wait-experiment.js`,在容器内实测):
|
| 41 |
+
|
| 42 |
+
| waitUntil 策略 | 耗时 |
|
| 43 |
+
|----------------|------|
|
| 44 |
+
| `setContent(load)` | **3–7ms** |
|
| 45 |
+
| `setContent(load + networkidle0)` | **~1970ms** |
|
| 46 |
+
| `setContent(load) + waitForNetworkIdle(500)` | ~505ms |
|
| 47 |
+
| 原代码两者都做 | **~2470ms** |
|
| 48 |
+
|
| 49 |
+
原代码既在 `setContent` 里 `networkidle0`,又在后面 `waitForNetworkIdle(500)`,对一个无外链资源的纯本地 HTML 白白等待 ~2.5s。
|
| 50 |
+
|
| 51 |
+
### 2.3 Widget 渲染每个 widget 启动一个浏览器
|
| 52 |
+
|
| 53 |
+
`renderWidgetPuppeteer()` / `_renderFullHtml()` 原代码**每个 widget 都 launch 一个新 Chromium**。一次导出 15 个 widget = 15 次浏览器启动。`WidgetRenderer._widgetBrowser` 单例存在但从未被使用(死代码)。
|
| 54 |
+
|
| 55 |
+
### 2.4 事件循环被同步操作阻塞
|
| 56 |
+
|
| 57 |
+
- 大 HTML 临时文件的 `fs.writeFileSync` / `fs.unlinkSync` 同步阻塞主线程
|
| 58 |
+
- 优化后改为 `fs.promises` 异步版本
|
| 59 |
+
|
| 60 |
+
### 2.5 响应被错误 JSON 序列化(我引入后立即修复)
|
| 61 |
+
|
| 62 |
+
`page.pdf()` 在 Puppeteer 24 返回 **`Uint8Array` 而非 Buffer**。`res.send(u8array)` 时 Express 不识别为二进制,会 `JSON.stringify` 成 `{"0":37,"1":80,...}`(约 13 倍体积)。原代码 `Buffer.from(pdfBuffer)` 正是为此。**任何修改都不能去掉这一步**。
|
| 63 |
+
|
| 64 |
+
---
|
| 65 |
+
|
| 66 |
+
## 三、实施的优化
|
| 67 |
+
|
| 68 |
+
### 3.1 新增 `browser-pool.js` — PDF 浏览器池
|
| 69 |
+
|
| 70 |
+
- 常驻 `PDF_POOL_SIZE` 个 Chromium(默认 4,env 可配)
|
| 71 |
+
- 每次请求 `acquire()` 一个浏览器 → 创建 page → 渲染 → 关闭 page → `release()` 归还
|
| 72 |
+
- 浏览器在 N 个任务后回收重建(`PDF_RECYCLE_AFTER=30`),防止 Chromium 长期运行内存膨胀
|
| 73 |
+
- 浏览器崩溃自动重建;池满时请求排队(`PDF_ACQUIRE_TIMEOUT_MS=120s`)
|
| 74 |
+
- 池大小 = 最大并发 PDF 数
|
| 75 |
+
|
| 76 |
+
### 3.2 Widget 渲染复用单例浏览器
|
| 77 |
+
|
| 78 |
+
`renderWidgetPuppeteer()` 与 `_renderFullHtml()` 改为使用 `getWidgetBrowser()` 共享浏览器(每个 widget 独立 page,渲染后只关 page 不关浏览器),并加了竞态保护与崩溃自动重建。
|
| 79 |
+
|
| 80 |
+
### 3.3 等待策略优化
|
| 81 |
+
|
| 82 |
+
- `setContent` / `goto`:`networkidle0` → `'load'`(每个请求省 ~2s)
|
| 83 |
+
- `waitForNetworkIdle`:idleTime 500 → 300(保留作为安全网)
|
| 84 |
+
- Widget 渲染同样处理
|
| 85 |
+
|
| 86 |
+
### 3.4 其他
|
| 87 |
+
|
| 88 |
+
- 大文件读写改异步 `fs.promises`
|
| 89 |
+
- 恢复 `res.send(Buffer.from(pdfBuffer))`(关键,见 2.5)
|
| 90 |
+
- 补上 `page.on('dialog')` 自动关弹窗(解决方案备忘录 24 的兜底,防止 XSS/异常 HTML 卡死)
|
| 91 |
+
- 配置全部 env 化:`PDF_POOL_SIZE` / `PDF_RECYCLE_AFTER` / `WIDGET_MAX_CONCURRENT`(docker-compose.yml 已配置:生产 2,测试 4)
|
| 92 |
+
|
| 93 |
+
---
|
| 94 |
+
|
| 95 |
+
## 四、压力测试数据
|
| 96 |
+
|
| 97 |
+
### 4.1 medium 负载(375KB 文本 + 3 张 base64 图)
|
| 98 |
+
|
| 99 |
+
| 并发 | 优化前 avg | 优化后 avg | 优化后 p95 | 优化后吞吐/min | 成功率 |
|
| 100 |
+
|------|-----------|-----------|-----------|---------------|--------|
|
| 101 |
+
| 1 | 4906ms | **1572ms** | 1843ms | 38 | 100% |
|
| 102 |
+
| 2 | 5283ms | **1882ms** | 2131ms | 62 | 100% |
|
| 103 |
+
| 4 | 5918ms | **2433ms** | 3752ms | 67 | 100% |
|
| 104 |
+
| 6 | 8919ms | **2758ms** | 4163ms | 81 | 100% |
|
| 105 |
+
| 8 | 未测 | **3569ms** | 5763ms | 83 | 100% |
|
| 106 |
+
|
| 107 |
+
### 4.2 small 负载(62KB,纯文本)
|
| 108 |
+
|
| 109 |
+
| 并发 | 优化前 avg | 优化后 avg | 优化后吞吐/min |
|
| 110 |
+
|------|-----------|-----------|---------------|
|
| 111 |
+
| 1 | 3380ms | **926ms** | 65 |
|
| 112 |
+
| 4 | — | **1513ms** | 117 |
|
| 113 |
+
| 8 | — | **2521ms** | 118 |
|
| 114 |
+
|
| 115 |
+
### 4.3 large 负载(1.24MB + 10 图,走临时文件路径)
|
| 116 |
+
|
| 117 |
+
| 并发 | 优化后 avg | 优化后吞吐/min | 成功率 |
|
| 118 |
+
|------|-----------|---------------|--------|
|
| 119 |
+
| 1 | 4988ms | 12 | 100% |
|
| 120 |
+
| 4 | 7748ms | 20 | 100% |
|
| 121 |
+
|
| 122 |
+
### 4.4 资源监控(optimized, medium, 4 浏览器常驻)
|
| 123 |
+
|
| 124 |
+
- CPU 峰值 ~392%(4 个浏览器同时渲染 ≈ 4 核忙)
|
| 125 |
+
- 内存峰值 **0.91GiB**,空闲 0.57GiB(4 个常驻浏览器 + Node)
|
| 126 |
+
- 相比优化前每请求新建浏览器、僵尸进程累积,资源完全受控
|
| 127 |
+
|
| 128 |
+
### 4.5 Widget 渲染
|
| 129 |
+
|
| 130 |
+
- 5 个 widget(4 chart + 1 mermaid)批量渲染:**7.03s,5/5 成功**,日志确认只启动 **1 次**浏览器(优化前每 widget 1 次)
|
| 131 |
+
- 单次 PDF 校验:输出为真实 PDF(`%PDF-1.4` 魔数,~113 页,1.6MB)
|
| 132 |
+
|
| 133 |
+
---
|
| 134 |
+
|
| 135 |
+
## 五、对 Hugging Face 免费版(2 vCPU)的预测
|
| 136 |
+
|
| 137 |
+
本机 16 核 docker 测出的容量是 **HF 的上界**,不能直接套用。按行业经验(每 page.pdf() 约占满 1 核):
|
| 138 |
+
|
| 139 |
+
- **HF 建议 `PDF_POOL_SIZE=2`**(docker-compose 已配置)→ 2 个用户同时导出,超出排队
|
| 140 |
+
- 单请求延迟在 HF 上会比本机高(CPU 弱),medium 预计 ~3–4s
|
| 141 |
+
- 吞吐预计 ~20–30/min(HF 2 核,CPU 是瓶颈)
|
| 142 |
+
- **结论**:当前 1000 用户规模绰绰有余;若未来用户数大幅增长,HF 免费版 2 vCPU 会成为瓶颈,需升级 `CPU Upgrade`(8 vCPU/32GB,$0.03/时)或使用 GPU Space
|
| 143 |
+
|
| 144 |
+
> ⚠️ **Docker 不能完全模拟 HuggingFace 机器**:CPU 核数(16 vs 2)差异巨大,PDF 渲染是 CPU 密集任务,因此本机测试结果只能作为相对对比(优化前后提升倍数),绝对并发数在 HF 上需按 2 vCPU 重新评估。
|
| 145 |
+
|
| 146 |
+
---
|
| 147 |
+
|
| 148 |
+
## 六、验证过的兼容性(避免复杂问题重现)
|
| 149 |
+
|
| 150 |
+
对照解决方案备忘录逐项确认优化未破坏:
|
| 151 |
+
|
| 152 |
+
| 备忘录 | 关注点 | 验证 |
|
| 153 |
+
|--------|--------|------|
|
| 154 |
+
| 24 (Puppeteer 超时/XSS) | dialog 卡死 setContent | ✅ 补回 `page.on('dialog')` 自动关闭 |
|
| 155 |
+
| 32 (Widget 渲染性能) | CDN 本地化失败、缓存方案 | ✅ 未重蹈 `setServerInterception` 覆辙;widget 单例浏览器按备忘录原设计实现 |
|
| 156 |
+
| 31 (表格串行 bug) | 前端导出流程 | ✅ 后端响应格式未变 |
|
| 157 |
+
| 05/06/26 (PDF/图表) | Shiki 高亮、Mermaid 版本固定 | ✅ 相关逻辑未改动,实测 PDF 与 widget 输出正确 |
|
| 158 |
+
|
| 159 |
+
---
|
| 160 |
+
|
| 161 |
+
## 七、后续建议(如需进一步优化)
|
| 162 |
+
|
| 163 |
+
1. **Shiki 高亮移入 Worker 线程**:大 HTML(>1MB)大量代码块时主线程阻塞;可放入 `worker_threads`。当前 medium/small 负载下 <100ms,非瓶颈。
|
| 164 |
+
2. **Node.js 多进程(cluster/PM2)**:充分利用多核处理主线程工作。注意每个 worker 需独立浏览器池。
|
| 165 |
+
3. **外部图片加载超时优化**:非 base64 图片失败时会等 15s 兜底(备忘录 03/04 相关)。可考虑对 http 图片缩短等待或降级。
|
| 166 |
+
4. **widget 渲染缓存**:备忘录 32 提到 CDN 响应内存缓存可提升 28%(此前实现丢失,本次未恢复,因为 `setServerInterception` 开销曾被证明为负收益;如需恢复需重新验证)。
|
| 167 |
+
5. **大规模扩展**:超过单机能力后,将 PDF 服务拆分为独立可水平扩展的服务(参考 medium 文章的直接 CDP + 队列架构)。
|
| 168 |
+
|
| 169 |
+
---
|
| 170 |
+
|
| 171 |
+
## 八、参考资料(行业最佳实践)
|
| 172 |
+
|
| 173 |
+
- [Puppeteer PDF: Common Problems and How to Fix](https://blog.pdfloom.com/puppeteer-pdf-problems/) — 浏览器池模式、内存 150-300MB/实例、并发限制
|
| 174 |
+
- [Optimizing Puppeteer PDF generation](https://www.codepasta.com/2024/04/19/optimizing-puppeteer-pdf-generation) — 并发数≈核数,队列限流
|
| 175 |
+
- [Designing a High-Performance HTML-to-PDF Service](https://medium.com/@harishrawat93/designing-a-high-performance-html-to-pdf-service-for-production-1a70099e3ccb) — 浏览器/页面池 + 队列架构
|
| 176 |
+
- [How to Fix Puppeteer Memory Leaks](https://www.grabbit.live/blog/puppeteer-memory-leak) — 浏览器按 N 任务回收(disposable Chromium)
|
| 177 |
+
- Hugging Face Spaces 免费版规格:CPU Basic = **2 vCPU / 16GB**([spaces-overview](https://huggingface.co/docs/hub/en/spaces-overview))
|
browser-pool.js
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* BrowserPool — a small, robust pool of reusable Puppeteer browsers.
|
| 3 |
+
*
|
| 4 |
+
* Why: launching a fresh Chromium per request is the #1 cause of poor
|
| 5 |
+
* PDF-export concurrency (each instance ~0.3–1.5s to start + 200–300MB RAM).
|
| 6 |
+
* This pool keeps `size` browsers warm; a job checks out a browser, does its
|
| 7 |
+
* work, then returns it. Browsers are recycled after `recycleAfter` jobs so a
|
| 8 |
+
* long-lived Chromium never accumulates too much memory.
|
| 9 |
+
*
|
| 10 |
+
* Design notes (industry best practice, verified by web research):
|
| 11 |
+
* - Concurrency = pool size. PDF/page rendering is CPU-bound; running more
|
| 12 |
+
* parallel jobs than the CPU count only degrades latency.
|
| 13 |
+
* - One job per browser at a time (no shared page juggling) keeps isolation
|
| 14 |
+
* and error handling trivial.
|
| 15 |
+
* - Browser crash / disconnect → replaced lazily on next acquire.
|
| 16 |
+
* - Callers must ALWAYS release in a finally block.
|
| 17 |
+
*/
|
| 18 |
+
|
| 19 |
+
const puppeteer = require('puppeteer');
|
| 20 |
+
|
| 21 |
+
const MAX_LAUNCH_FAILURES_PER_SLOT = 3;
|
| 22 |
+
|
| 23 |
+
class BrowserPool {
|
| 24 |
+
/**
|
| 25 |
+
* @param {object} opts
|
| 26 |
+
* @param {string} opts.name label for logs
|
| 27 |
+
* @param {number} opts.size number of browsers to keep
|
| 28 |
+
* @param {object} opts.launchOptions puppeteer.launch() options
|
| 29 |
+
* @param {number} [opts.recycleAfter] jobs per browser before recycle (default 30)
|
| 30 |
+
* @param {number} [opts.acquireTimeoutMs] how long a waiter waits for a free browser (default 120000)
|
| 31 |
+
* @param {Function} [opts.log]
|
| 32 |
+
*/
|
| 33 |
+
constructor(opts) {
|
| 34 |
+
this.name = opts.name || 'pool';
|
| 35 |
+
this.size = Math.max(1, Math.min(16, Math.floor(opts.size) || 1));
|
| 36 |
+
this.launchOptions = opts.launchOptions || {};
|
| 37 |
+
this.recycleAfter = opts.recycleAfter || 30;
|
| 38 |
+
this.acquireTimeoutMs = opts.acquireTimeoutMs || 120000;
|
| 39 |
+
this.log = opts.log || (() => {});
|
| 40 |
+
this._slots = [];
|
| 41 |
+
this._waiters = [];
|
| 42 |
+
this.stats = { acquires: 0, launches: 0, recycles: 0, waits: 0, waitTimeouts: 0, errors: 0 };
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
_log(msg) {
|
| 46 |
+
this.log(`[POOL:${this.name}] ${msg}`);
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
_newSlot() {
|
| 50 |
+
return { browser: null, jobs: 0, available: true, closed: false, launching: false, launchFailures: 0 };
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
async _launch() {
|
| 54 |
+
this.stats.launches++;
|
| 55 |
+
this._log(`launching browser (total launches=${this.stats.launches})`);
|
| 56 |
+
const browser = await puppeteer.launch(this.launchOptions);
|
| 57 |
+
const slot = this._newSlot();
|
| 58 |
+
slot.browser = browser;
|
| 59 |
+
browser.on('disconnected', () => {
|
| 60 |
+
slot.closed = true;
|
| 61 |
+
slot.available = true;
|
| 62 |
+
this._log('browser disconnected (crash/kill); slot marked closed');
|
| 63 |
+
});
|
| 64 |
+
return slot;
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
_pump() {
|
| 68 |
+
// 1. Hand free browsers to waiting jobs.
|
| 69 |
+
while (this._waiters.length > 0) {
|
| 70 |
+
const slot = this._slots.find((s) => s.available && !s.closed && s.browser);
|
| 71 |
+
if (!slot) break;
|
| 72 |
+
const waiter = this._waiters.shift();
|
| 73 |
+
clearTimeout(waiter.timer);
|
| 74 |
+
slot.available = false;
|
| 75 |
+
// IMPORTANT: resolve with the wrapper ({browser, release}), same shape
|
| 76 |
+
// as the warm-path return in acquire(). Resolving with the raw slot made
|
| 77 |
+
// callers' `acquired.release` undefined → slots never released → pool
|
| 78 |
+
// deadlocked (3/4 busy forever).
|
| 79 |
+
waiter.resolve(this._wrap(slot));
|
| 80 |
+
}
|
| 81 |
+
// 2. Grow/refill the pool (launch one browser per pump pass).
|
| 82 |
+
// 必须「无论是否有 waiter」都启动空槽位:否则第 size 个槽位被 push 时
|
| 83 |
+
// slots.length === size 使 `length < size` 为假,该槽位永远不会被启动,
|
| 84 |
+
// 池实际并发只有 size-1,第 N 个任务必须等第一个任务释放浏览器后才能执行
|
| 85 |
+
// (实测 3 个 widget 的批次因此从 ~2.5s 被串行拖到 ~5s)。
|
| 86 |
+
// 浏览器回收(release 中 splice 移除槽位)后也要补位,保证池始终补齐到 size。
|
| 87 |
+
if (this._slots.length < this.size) {
|
| 88 |
+
this._slots.push(this._newSlot());
|
| 89 |
+
}
|
| 90 |
+
const empty = this._slots.find((s) => !s.browser && !s.launching);
|
| 91 |
+
if (empty) {
|
| 92 |
+
empty.launching = true;
|
| 93 |
+
this._launch().then((slot) => {
|
| 94 |
+
const i = this._slots.indexOf(empty);
|
| 95 |
+
if (i === -1) { slot.browser.close().catch(() => {}); return; }
|
| 96 |
+
this._slots[i] = slot;
|
| 97 |
+
this._pump();
|
| 98 |
+
}).catch((err) => {
|
| 99 |
+
this.stats.errors++;
|
| 100 |
+
this._log(`browser launch failed: ${err.message}`);
|
| 101 |
+
empty.launchFailures++;
|
| 102 |
+
const i = this._slots.indexOf(empty);
|
| 103 |
+
if (i !== -1) {
|
| 104 |
+
if (empty.launchFailures >= MAX_LAUNCH_FAILURES_PER_SLOT) {
|
| 105 |
+
this._slots.splice(i, 1);
|
| 106 |
+
const waiter = this._waiters.shift();
|
| 107 |
+
if (waiter) {
|
| 108 |
+
clearTimeout(waiter.timer);
|
| 109 |
+
waiter.reject(new Error(`[POOL:${this.name}] browser launch failed: ${err.message}`));
|
| 110 |
+
}
|
| 111 |
+
} else {
|
| 112 |
+
empty.launching = false; // allow retry
|
| 113 |
+
}
|
| 114 |
+
}
|
| 115 |
+
this._pump();
|
| 116 |
+
});
|
| 117 |
+
}
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
/**
|
| 121 |
+
* Check out a browser slot for one job.
|
| 122 |
+
* @returns {Promise<{browser: object, release: Function}>}
|
| 123 |
+
*/
|
| 124 |
+
async acquire() {
|
| 125 |
+
this.stats.acquires++;
|
| 126 |
+
const warm = this._slots.find((s) => s.available && !s.closed && s.browser);
|
| 127 |
+
if (warm) {
|
| 128 |
+
warm.available = false;
|
| 129 |
+
this._log(`acquire: warm slot (inUse=${this._inUse()}/${this.size})`);
|
| 130 |
+
return this._wrap(warm);
|
| 131 |
+
}
|
| 132 |
+
// Reserve capacity to grow the pool.
|
| 133 |
+
if (this._slots.length < this.size) {
|
| 134 |
+
this._slots.push(this._newSlot());
|
| 135 |
+
this._pump();
|
| 136 |
+
}
|
| 137 |
+
this.stats.waits++;
|
| 138 |
+
this._log(`acquire: no free slot, queued (inUse=${this._inUse()}/${this.size})`);
|
| 139 |
+
return new Promise((resolve, reject) => {
|
| 140 |
+
const timer = setTimeout(() => {
|
| 141 |
+
const i = this._waiters.indexOf(waiter);
|
| 142 |
+
if (i !== -1) this._waiters.splice(i, 1);
|
| 143 |
+
this.stats.waitTimeouts++;
|
| 144 |
+
this._log(`acquire timed out after ${this.acquireTimeoutMs}ms`);
|
| 145 |
+
reject(new Error(`[POOL:${this.name}] no free browser within ${this.acquireTimeoutMs}ms (busy=${this._inUse()}/${this.size})`));
|
| 146 |
+
}, this.acquireTimeoutMs);
|
| 147 |
+
const waiter = { resolve, reject, timer };
|
| 148 |
+
this._waiters.push(waiter);
|
| 149 |
+
this._pump();
|
| 150 |
+
});
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
_inUse() {
|
| 154 |
+
return this._slots.filter((s) => !s.available).length;
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
_wrap(slot) {
|
| 158 |
+
return {
|
| 159 |
+
browser: slot.browser,
|
| 160 |
+
release: async () => {
|
| 161 |
+
slot.jobs++;
|
| 162 |
+
if (slot.closed || slot.jobs >= this.recycleAfter) {
|
| 163 |
+
this.stats.recycles++;
|
| 164 |
+
this._log(`recycling browser after ${slot.jobs} jobs (recycles=${this.stats.recycles})`);
|
| 165 |
+
const i = this._slots.indexOf(slot);
|
| 166 |
+
if (i !== -1) this._slots.splice(i, 1);
|
| 167 |
+
try { await slot.browser.close(); } catch (e) {}
|
| 168 |
+
slot.closed = true;
|
| 169 |
+
} else {
|
| 170 |
+
slot.available = true;
|
| 171 |
+
this._log(`released browser (jobs=${slot.jobs}, inUse=${this._inUse()}/${this.size})`);
|
| 172 |
+
}
|
| 173 |
+
this._pump();
|
| 174 |
+
},
|
| 175 |
+
};
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
async close() {
|
| 179 |
+
const slots = this._slots.splice(0);
|
| 180 |
+
for (const s of slots) {
|
| 181 |
+
if (s.browser) { try { await s.browser.close(); } catch (e) {} }
|
| 182 |
+
}
|
| 183 |
+
for (const w of this._waiters.splice(0)) {
|
| 184 |
+
clearTimeout(w.timer);
|
| 185 |
+
w.reject(new Error(`[POOL:${this.name}] pool closed`));
|
| 186 |
+
}
|
| 187 |
+
}
|
| 188 |
+
}
|
| 189 |
+
|
| 190 |
+
module.exports = { BrowserPool };
|
docker-compose.yml
CHANGED
|
@@ -1,5 +1,3 @@
|
|
| 1 |
-
version: '3.8'
|
| 2 |
-
|
| 3 |
services:
|
| 4 |
pdf-prod:
|
| 5 |
build: .
|
|
@@ -10,6 +8,10 @@ services:
|
|
| 10 |
environment:
|
| 11 |
- NODE_ENV=production
|
| 12 |
- PORT=7860
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
logging:
|
| 14 |
driver: "json-file"
|
| 15 |
options:
|
|
@@ -21,12 +23,17 @@ services:
|
|
| 21 |
container_name: pdf-test
|
| 22 |
restart: always
|
| 23 |
ports:
|
| 24 |
-
- "
|
| 25 |
environment:
|
| 26 |
- NODE_ENV=test
|
| 27 |
- PORT=7860
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
logging:
|
| 29 |
driver: "json-file"
|
| 30 |
options:
|
| 31 |
max-size: "10m"
|
| 32 |
max-file: "3"
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
services:
|
| 2 |
pdf-prod:
|
| 3 |
build: .
|
|
|
|
| 8 |
environment:
|
| 9 |
- NODE_ENV=production
|
| 10 |
- PORT=7860
|
| 11 |
+
# Hugging Face free tier = 2 vCPU / 16GB → PDF browser pool of 2.
|
| 12 |
+
# page.pdf() is CPU-bound; pool size = max concurrent PDF jobs.
|
| 13 |
+
- PDF_POOL_SIZE=2
|
| 14 |
+
- WIDGET_MAX_CONCURRENT=3
|
| 15 |
logging:
|
| 16 |
driver: "json-file"
|
| 17 |
options:
|
|
|
|
| 23 |
container_name: pdf-test
|
| 24 |
restart: always
|
| 25 |
ports:
|
| 26 |
+
- "17861:7860"
|
| 27 |
environment:
|
| 28 |
- NODE_ENV=test
|
| 29 |
- PORT=7860
|
| 30 |
+
# Local test machine: multi-core → larger pool for finding max capacity.
|
| 31 |
+
# Tune with: docker compose up -d --build pdf-test (edit here or override)
|
| 32 |
+
- PDF_POOL_SIZE=4
|
| 33 |
+
- WIDGET_MAX_CONCURRENT=3
|
| 34 |
logging:
|
| 35 |
driver: "json-file"
|
| 36 |
options:
|
| 37 |
max-size: "10m"
|
| 38 |
max-file: "3"
|
| 39 |
+
|
lib/chart.umd.js
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
make/_clean.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
|
| 3 |
+
with open('D:/workAI/OpenCode/WorkSpace/backend-service/server.js', 'r', encoding='utf-8') as f:
|
| 4 |
+
content = f.read()
|
| 5 |
+
|
| 6 |
+
# Clean _renderFullHtml: replace entire debug-heavy section with clean version
|
| 7 |
+
# Find from setViewport to dataUrl assignment
|
| 8 |
+
pattern = r"( await page\.setViewport\(\{ width: 800, height: 500 \}\);).*?(const dataUrl = `data:image/png;base64,\$\{screenshot\.toString\('base64'\)\}`;)"
|
| 9 |
+
replacement = r""" await page.setViewport({ width: 800, height: 500 });
|
| 10 |
+
|
| 11 |
+
await page.setContent(htmlContent, { waitUntil: 'networkidle0', timeout: 15000 });
|
| 12 |
+
|
| 13 |
+
const rendered = await page.waitForFunction(
|
| 14 |
+
() => window._widgetRendered === true,
|
| 15 |
+
{ timeout: renderTimeout }
|
| 16 |
+
).catch(() => null);
|
| 17 |
+
|
| 18 |
+
if (!rendered) {
|
| 19 |
+
console.log(`[WIDGET] _renderFullHtml timeout after ${renderTimeout}ms, trying screenshot anyway`);
|
| 20 |
+
await new Promise(r => setTimeout(r, 1000));
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
const container = await page.$('#widget-container');
|
| 24 |
+
let screenshot;
|
| 25 |
+
if (container) {
|
| 26 |
+
screenshot = await container.screenshot({ type: 'png', omitBackground: false });
|
| 27 |
+
} else {
|
| 28 |
+
screenshot = await page.screenshot({ type: 'png', fullPage: true, omitBackground: false });
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
const dataUrl = `data:image/png;base64,${screenshot.toString('base64')}`;"""
|
| 32 |
+
|
| 33 |
+
content = re.sub(pattern, replacement, content, flags=re.DOTALL)
|
| 34 |
+
print("CLEANED: _renderFullHtml debug blocks")
|
| 35 |
+
|
| 36 |
+
# Remove debugId from _renderFullHtml call
|
| 37 |
+
content = content.replace(
|
| 38 |
+
"return this._renderFullHtml(html, chartTitle, 8000, chartType, debugId);",
|
| 39 |
+
"return this._renderFullHtml(html, chartTitle, 8000, chartType);"
|
| 40 |
+
)
|
| 41 |
+
print("CLEANED: Removed debugId from _renderFullHtml call")
|
| 42 |
+
|
| 43 |
+
# Remove debugId from method signature
|
| 44 |
+
content = content.replace(
|
| 45 |
+
"async _renderFullHtml(htmlContent, title, renderTimeout, chartType, debugId)",
|
| 46 |
+
"async _renderFullHtml(htmlContent, title, renderTimeout, chartType)"
|
| 47 |
+
)
|
| 48 |
+
print("CLEANED: Removed debugId from _renderFullHtml signature")
|
| 49 |
+
|
| 50 |
+
# Remove redundant fs require in renderChatGPTChart (already partially cleaned)
|
| 51 |
+
old_debug = """ const html = this.buildChatGPTChartHtml(chartType, series, data, xKey, nameKey, valueKey, formattedData, axes, chartTitle, chartDesc);
|
| 52 |
+
|
| 53 |
+
// DEBUG: Write HTML to disk for inspection
|
| 54 |
+
const fs = require('fs');
|
| 55 |
+
const debugId = `${chartType}-${Date.now()}`;
|
| 56 |
+
fs.writeFileSync(`/tmp/chart-debug-${debugId}.html`, html);
|
| 57 |
+
|
| 58 |
+
// DO NOT use renderWidgetPuppeteer"""
|
| 59 |
+
|
| 60 |
+
if old_debug in content:
|
| 61 |
+
content = content.replace(old_debug,
|
| 62 |
+
" const html = this.buildChatGPTChartHtml(chartType, series, data, xKey, nameKey, valueKey, formattedData, axes, chartTitle, chartDesc);")
|
| 63 |
+
print("CLEANED: Removed /tmp debug HTML write")
|
| 64 |
+
else:
|
| 65 |
+
print("INFO: /tmp debug write already cleaned or not found")
|
| 66 |
+
|
| 67 |
+
with open('D:/workAI/OpenCode/WorkSpace/backend-service/server.js', 'w', encoding='utf-8') as f:
|
| 68 |
+
f.write(content)
|
| 69 |
+
|
| 70 |
+
print("Done.")
|
make/_clean2.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
|
| 3 |
+
with open('D:/workAI/OpenCode/WorkSpace/backend-service/server.js', 'r', encoding='utf-8') as f:
|
| 4 |
+
content = f.read()
|
| 5 |
+
|
| 6 |
+
# 1. Remove dead totalWaitTime/baseWaitTime/sizeWaitTime variables (around line 240)
|
| 7 |
+
old_wait = """ const baseWaitTime = imgCount > 0 ? Math.min(500 + imgCount * 400, 6000) : 500;
|
| 8 |
+
const sizeWaitTime = imgSizeMB > 0 ? Math.min(imgSizeMB * 100, 3000) : 0;
|
| 9 |
+
const totalWaitTime = Math.min(baseWaitTime + sizeWaitTime, 8000);"""
|
| 10 |
+
if old_wait in content:
|
| 11 |
+
content = content.replace(old_wait, "")
|
| 12 |
+
print("CLEANED: Removed dead baseWaitTime/sizeWaitTime/totalWaitTime variables")
|
| 13 |
+
else:
|
| 14 |
+
print("INFO: Wait time variables already cleaned or different")
|
| 15 |
+
|
| 16 |
+
# 2. Remove dead _rawToFormatted object (around line 1303)
|
| 17 |
+
# This is a block that builds a mapping but never uses it
|
| 18 |
+
# Find the pattern: var _rawToFormatted = {}; ... (until it's no longer referenced)
|
| 19 |
+
raw_to_fmt_start = " var _rawToFormatted = {};"
|
| 20 |
+
raw_to_fmt_end = " }" # end of the forEach that populates it
|
| 21 |
+
idx = content.find(raw_to_fmt_start)
|
| 22 |
+
if idx >= 0:
|
| 23 |
+
# Find the end of this block - it's the forEach that populates _rawToFormatted
|
| 24 |
+
# The next section starts with "var chartSeries" or similar
|
| 25 |
+
# Find the closing brace of the forEach
|
| 26 |
+
search_from = idx
|
| 27 |
+
brace_count = 0
|
| 28 |
+
end_idx = -1
|
| 29 |
+
in_block = False
|
| 30 |
+
i = idx
|
| 31 |
+
while i < len(content):
|
| 32 |
+
if content[i:i+2] == "// ":
|
| 33 |
+
# Skip to end of line
|
| 34 |
+
newline = content.find("\n", i)
|
| 35 |
+
if newline < 0: break
|
| 36 |
+
i = newline + 1
|
| 37 |
+
continue
|
| 38 |
+
if content[i] == "{":
|
| 39 |
+
brace_count += 1
|
| 40 |
+
in_block = True
|
| 41 |
+
elif content[i] == "}":
|
| 42 |
+
brace_count -= 1
|
| 43 |
+
if in_block and brace_count == 0:
|
| 44 |
+
end_idx = i + 1
|
| 45 |
+
break
|
| 46 |
+
i += 1
|
| 47 |
+
|
| 48 |
+
if end_idx > 0:
|
| 49 |
+
block = content[idx:end_idx]
|
| 50 |
+
# Check if _rawToFormatted is used after this block
|
| 51 |
+
remaining = content[end_idx:]
|
| 52 |
+
if "_rawToFormatted" not in remaining:
|
| 53 |
+
content = content[:idx] + content[end_idx:]
|
| 54 |
+
print("CLEANED: Removed dead _rawToFormatted object")
|
| 55 |
+
else:
|
| 56 |
+
print("WARNING: _rawToFormatted is still referenced, skipping")
|
| 57 |
+
else:
|
| 58 |
+
print("WARNING: Could not find end of _rawToFormatted block")
|
| 59 |
+
else:
|
| 60 |
+
print("INFO: _rawToFormatted already cleaned or not found")
|
| 61 |
+
|
| 62 |
+
# 3. Remove dead pieActualDataKey
|
| 63 |
+
if "var pieActualDataKey = valueKey" in content:
|
| 64 |
+
content = content.replace("var pieActualDataKey = valueKey || chartSeries[0]?.dataKey || 'value';\n", "")
|
| 65 |
+
print("CLEANED: Removed dead pieActualDataKey")
|
| 66 |
+
else:
|
| 67 |
+
print("INFO: pieActualDataKey already cleaned or not found")
|
| 68 |
+
|
| 69 |
+
# 4. Fix double-count regex pattern for new C(
|
| 70 |
+
if "new\\\\s+C\\\\s*\\\\(" in content:
|
| 71 |
+
content = content.replace(
|
| 72 |
+
"(widgetHtml.match(/new\\\\s+C\\\\s*\\\\(/gi) || []).length +\\n ",
|
| 73 |
+
""
|
| 74 |
+
)
|
| 75 |
+
print("CLEANED: Removed double-count new C( regex")
|
| 76 |
+
else:
|
| 77 |
+
# Try without double escaping
|
| 78 |
+
if "new\\\\s+C\\\\s*\\\\(" in content:
|
| 79 |
+
content = content.replace(
|
| 80 |
+
"(widgetHtml.match(/new\\s+C\\s*\\(/gi) || []).length +\n ",
|
| 81 |
+
""
|
| 82 |
+
)
|
| 83 |
+
print("CLEANED: Removed double-count new C( regex (alt)")
|
| 84 |
+
else:
|
| 85 |
+
print("INFO: Double-count regex not found or different format")
|
| 86 |
+
|
| 87 |
+
# 5. Remove duplicate COLORS entries
|
| 88 |
+
old_colors = """ const COLORS = [
|
| 89 |
+
'#339CFF', '#40C977', '#FF8549', '#FFD240',
|
| 90 |
+
'#339CFF', '#40C977', '#FF8549', '#FFD240',
|
| 91 |
+
'#339CFF', '#40C977', '#FF8549', '#FFD240',
|
| 92 |
+
];"""
|
| 93 |
+
new_colors = """ const COLORS = ['#339CFF', '#40C977', '#FF8549', '#FFD240'];"""
|
| 94 |
+
if old_colors in content:
|
| 95 |
+
content = content.replace(old_colors, new_colors)
|
| 96 |
+
print("CLEANED: Removed duplicate COLORS entries")
|
| 97 |
+
else:
|
| 98 |
+
print("INFO: COLORS already cleaned or different format")
|
| 99 |
+
|
| 100 |
+
# 6. Remove %%INJECTED_SCRIPTS%% placeholder (always replaced with empty string)
|
| 101 |
+
old_inject = """.replace('%%INJECTED_SCRIPTS%%', '');"""
|
| 102 |
+
if old_inject in content:
|
| 103 |
+
content = content.replace(old_inject, ";")
|
| 104 |
+
print("CLEANED: Removed %%INJECTED_SCRIPTS%% placeholder")
|
| 105 |
+
else:
|
| 106 |
+
print("INFO: %%INJECTED_SCRIPTS%% already cleaned or not found")
|
| 107 |
+
|
| 108 |
+
# 7. Clean up verbose PIE DATA logging
|
| 109 |
+
old_pie_log = """ if (chartType === 'pie' || chartType === 'donut') {
|
| 110 |
+
console.log(`[WIDGET] renderChatGPTChart PIE DATA: data=${JSON.stringify(data)}, formattedData=${JSON.stringify(formattedData)}`);
|
| 111 |
+
}"""
|
| 112 |
+
if old_pie_log in content:
|
| 113 |
+
content = content.replace(old_pie_log, "")
|
| 114 |
+
print("CLEANED: Removed verbose PIE DATA logging")
|
| 115 |
+
else:
|
| 116 |
+
print("INFO: PIE DATA logging already cleaned")
|
| 117 |
+
|
| 118 |
+
# 8. Clean up nameKey/valueKey verbose logging
|
| 119 |
+
old_kv_log = """ if (nameKey) console.log(`[WIDGET] renderChatGPTChart: nameKey=${nameKey}, valueKey=${valueKey}`);"""
|
| 120 |
+
if old_kv_log in content:
|
| 121 |
+
content = content.replace(old_kv_log, "")
|
| 122 |
+
print("CLEANED: Removed verbose nameKey/valueKey logging")
|
| 123 |
+
else:
|
| 124 |
+
print("INFO: nameKey/valueKey logging already cleaned")
|
| 125 |
+
|
| 126 |
+
# 9. Remove [WIDGET] DEBUG widgetHtml length logging
|
| 127 |
+
old_widget_log = " console.log(`[WIDGET] DEBUG widgetHtml length=` + widgetHtml.length + `, first 500 chars: ` + widgetHtml.substring(0, 500));"
|
| 128 |
+
if old_widget_log in content:
|
| 129 |
+
content = content.replace(old_widget_log, "")
|
| 130 |
+
print("CLEANED: Removed widgetHtml length debug logging")
|
| 131 |
+
else:
|
| 132 |
+
print("INFO: widgetHtml length logging already cleaned")
|
| 133 |
+
|
| 134 |
+
# 10. Remove [WIDGET] extractChartConfig DEBUG logging
|
| 135 |
+
old_extract_log = " console.log(`[WIDGET] extractChartConfig: DEBUG widgetHtml (first 800 chars): ` + widgetHtml.substring(0, 800));"
|
| 136 |
+
if old_extract_log in content:
|
| 137 |
+
content = content.replace(old_extract_log, "")
|
| 138 |
+
print("CLEANED: Removed extractChartConfig DEBUG logging")
|
| 139 |
+
else:
|
| 140 |
+
print("INFO: extractChartConfig DEBUG logging already cleaned")
|
| 141 |
+
|
| 142 |
+
# 11. Remove [WIDGET] renderChart DEBUG logging
|
| 143 |
+
old_render_log = " console.log(`[WIDGET] renderChart DEBUG: widgetHtml length=` + widgetHtml.length + `, first 500 chars: ` + widgetHtml.substring(0, 500));"
|
| 144 |
+
if old_render_log in content:
|
| 145 |
+
content = content.replace(old_render_log, "")
|
| 146 |
+
print("CLEANED: Removed renderChart DEBUG logging")
|
| 147 |
+
else:
|
| 148 |
+
print("INFO: renderChart DEBUG logging already cleaned")
|
| 149 |
+
|
| 150 |
+
with open('D:/workAI/OpenCode/WorkSpace/backend-service/server.js', 'w', encoding='utf-8') as f:
|
| 151 |
+
f.write(content)
|
| 152 |
+
|
| 153 |
+
print("\nDone.")
|
make/_clean3.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
|
| 3 |
+
with open('D:/workAI/OpenCode/WorkSpace/backend-service/server.js', 'r', encoding='utf-8') as f:
|
| 4 |
+
content = f.read()
|
| 5 |
+
|
| 6 |
+
# 1. Remove dead _rawToFormatted block (lines 1171-1188)
|
| 7 |
+
# Find and remove the entire block from comment to closing brace
|
| 8 |
+
pattern = r" // .*?Build format map from chartData.*?\n var _rawToFormatted = \{\};.*?\n \}\n\n // Build series elements"
|
| 9 |
+
replacement = " // Build series elements"
|
| 10 |
+
new_content = re.sub(pattern, replacement, content, flags=re.DOTALL)
|
| 11 |
+
if new_content != content:
|
| 12 |
+
content = new_content
|
| 13 |
+
print("CLEANED: Removed dead _rawToFormatted block")
|
| 14 |
+
else:
|
| 15 |
+
# Try a simpler pattern
|
| 16 |
+
idx = content.find("var _rawToFormatted = {};")
|
| 17 |
+
if idx >= 0:
|
| 18 |
+
# Go back to find the comment before it
|
| 19 |
+
comment_start = content.rfind(" // ", 0, idx)
|
| 20 |
+
# Find the end - next " // Build series elements"
|
| 21 |
+
end_marker = " // Build series elements"
|
| 22 |
+
end_idx = content.find(end_marker, idx)
|
| 23 |
+
if end_idx > 0 and comment_start > 0:
|
| 24 |
+
content = content[:comment_start] + "\n" + content[end_idx:]
|
| 25 |
+
print("CLEANED: Removed dead _rawToFormatted block (manual)")
|
| 26 |
+
else:
|
| 27 |
+
print(f"WARNING: Could not find boundaries. comment_start={comment_start}, end_idx={end_idx}")
|
| 28 |
+
else:
|
| 29 |
+
print("INFO: _rawToFormatted not found")
|
| 30 |
+
|
| 31 |
+
# 2. Remove all [AXIS-DEBUG] console.log from generated HTML
|
| 32 |
+
# These are in the client-side code embedded in buildChatGPTChartHtml
|
| 33 |
+
axis_debug_count = len(re.findall(r"console\.log\('\[AXIS-DEBUG\]", content))
|
| 34 |
+
if axis_debug_count > 0:
|
| 35 |
+
# Remove console.log('[AXIS-DEBUG]...'); lines
|
| 36 |
+
content = re.sub(r" console\.log\('\[AXIS-DEBUG\].*?\);\n", "", content)
|
| 37 |
+
content = re.sub(r" console\.log\('\[AXIS-DEBUG\].*?\);\n", "", content)
|
| 38 |
+
print(f"CLEANED: Removed {axis_debug_count} [AXIS-DEBUG] console.log statements")
|
| 39 |
+
else:
|
| 40 |
+
print("INFO: No [AXIS-DEBUG] logs found")
|
| 41 |
+
|
| 42 |
+
# 3. Remove [WIDGET] extractChartConfig FAIL logging
|
| 43 |
+
idx = content.find("console.log(`[WIDGET] extractChartConfig: FAIL")
|
| 44 |
+
if idx >= 0:
|
| 45 |
+
line_end = content.find(";", idx) + 1
|
| 46 |
+
content = content[:idx] + content[line_end:]
|
| 47 |
+
print("CLEANED: Removed extractChartConfig FAIL logging")
|
| 48 |
+
else:
|
| 49 |
+
print("INFO: extractChartConfig FAIL logging not found")
|
| 50 |
+
|
| 51 |
+
# 4. Remove redundant const fs = require inside renderChatGPTChart
|
| 52 |
+
# Check for local fs requires (should not exist since fs is at top)
|
| 53 |
+
fs_local_count = len(re.findall(r"const fs = require\('fs'\)", content))
|
| 54 |
+
if fs_local_count > 1:
|
| 55 |
+
# Keep the first (top-level), remove duplicates
|
| 56 |
+
first = content.find("const fs = require('fs')")
|
| 57 |
+
content = content[:first] + "__FS_MARKER__" + content[first+len("const fs = require('fs')"):]
|
| 58 |
+
content = content.replace("const fs = require('fs')", "")
|
| 59 |
+
content = content.replace("__FS_MARKER__", "const fs = require('fs')")
|
| 60 |
+
print(f"CLEANED: Removed {fs_local_count - 1} duplicate fs requires")
|
| 61 |
+
else:
|
| 62 |
+
print(f"INFO: fs requires count = {fs_local_count} (OK)")
|
| 63 |
+
|
| 64 |
+
# 5. Clean up the comment about Chart.js detection (remove redundant pattern)
|
| 65 |
+
# The new\s+C\s*\( pattern double-counts
|
| 66 |
+
pattern = r"\(widgetHtml\.match\(/new\\s\+C\\s\*\\\(/gi\) \|\| \[\]\)\.length \+\s*\n\s*"
|
| 67 |
+
if re.search(pattern, content):
|
| 68 |
+
content = re.sub(pattern, "", content)
|
| 69 |
+
print("CLEANED: Removed double-count new C( regex")
|
| 70 |
+
else:
|
| 71 |
+
print("INFO: Double-count regex not found")
|
| 72 |
+
|
| 73 |
+
# 6. Remove the pieActualDataKey dead variable
|
| 74 |
+
content = content.replace("var pieActualDataKey = valueKey || chartSeries[0]?.dataKey || 'value';\n", "")
|
| 75 |
+
print("CLEANED: Removed dead pieActualDataKey (second pass)")
|
| 76 |
+
|
| 77 |
+
with open('D:/workAI/OpenCode/WorkSpace/backend-service/server.js', 'w', encoding='utf-8') as f:
|
| 78 |
+
f.write(content)
|
| 79 |
+
|
| 80 |
+
print("\nDone.")
|
make/_clean_final.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
|
| 3 |
+
with open('D:/workAI/OpenCode/WorkSpace/backend-service/server.js', 'r', encoding='utf-8') as f:
|
| 4 |
+
lines = f.readlines()
|
| 5 |
+
|
| 6 |
+
new_lines = []
|
| 7 |
+
skip_until = -1 # Line index to skip until
|
| 8 |
+
|
| 9 |
+
for i, line in enumerate(lines):
|
| 10 |
+
# Skip lines until skip_until
|
| 11 |
+
if i < skip_until:
|
| 12 |
+
continue
|
| 13 |
+
|
| 14 |
+
# 1. Remove dead baseWaitTime/sizeWaitTime/totalWaitTime (lines ~240-242)
|
| 15 |
+
if 'const baseWaitTime = imgCount > 0' in line:
|
| 16 |
+
continue
|
| 17 |
+
if 'const sizeWaitTime = imgSizeMB > 0' in line:
|
| 18 |
+
continue
|
| 19 |
+
if 'const totalWaitTime = Math.min(baseWaitTime' in line:
|
| 20 |
+
continue
|
| 21 |
+
|
| 22 |
+
# 2. Remove /tmp debug HTML write block (lines ~943-946)
|
| 23 |
+
if line.strip() == '// DEBUG: Write HTML to disk for inspection':
|
| 24 |
+
continue
|
| 25 |
+
if line.strip() == "const fs = require('fs');" and i > 900 and i < 1000:
|
| 26 |
+
# Only remove the one inside renderChatGPTChart, not the top-level import
|
| 27 |
+
# Check context: previous line should mention DEBUG
|
| 28 |
+
if i > 0 and 'DEBUG' in lines[i-1]:
|
| 29 |
+
continue
|
| 30 |
+
else:
|
| 31 |
+
new_lines.append(line)
|
| 32 |
+
continue
|
| 33 |
+
if 'fs.writeFileSync(`/tmp/chart-debug-' in line:
|
| 34 |
+
continue
|
| 35 |
+
if line.strip() == '// DO NOT use renderWidgetPuppeteer — it wraps content in a <div> which breaks':
|
| 36 |
+
continue
|
| 37 |
+
if 'the complete HTML document (script tags in <head> won' in line:
|
| 38 |
+
continue
|
| 39 |
+
if '// Instead, use page.setContent directly with the full HTML.' in line:
|
| 40 |
+
continue
|
| 41 |
+
|
| 42 |
+
# 3. Remove debugId from _renderFullHtml call and signature
|
| 43 |
+
if 'return this._renderFullHtml(html, chartTitle, 8000, chartType, debugId)' in line:
|
| 44 |
+
new_lines.append(line.replace(', debugId', ''))
|
| 45 |
+
continue
|
| 46 |
+
if 'async _renderFullHtml(htmlContent, title, renderTimeout, chartType, debugId)' in line:
|
| 47 |
+
new_lines.append(line.replace(', debugId', ''))
|
| 48 |
+
continue
|
| 49 |
+
|
| 50 |
+
# 4. Remove viewport comment + CDN + console forwarding
|
| 51 |
+
if '// CRITICAL: Use a compact viewport height to avoid excessive whitespace' in line:
|
| 52 |
+
continue
|
| 53 |
+
if '// in fullPage screenshots. Width 800 gives enough room for 650px chart.' in line:
|
| 54 |
+
continue
|
| 55 |
+
if '// CDN failure listener for debugging' in line:
|
| 56 |
+
# Skip this line and the next 3 lines
|
| 57 |
+
skip_until = i + 4
|
| 58 |
+
continue
|
| 59 |
+
if '// Forward page console logs to Node.js console for debug' in line:
|
| 60 |
+
# Skip this line and the next 6 lines
|
| 61 |
+
skip_until = i + 7
|
| 62 |
+
continue
|
| 63 |
+
if '// Set complete HTML document directly' in line:
|
| 64 |
+
continue
|
| 65 |
+
|
| 66 |
+
# 5. Remove containerInfo debug block
|
| 67 |
+
if '// ── DEBUG: Log what\'s actually rendered in the container' in line:
|
| 68 |
+
# Skip until the closing console.log of this block
|
| 69 |
+
skip_until = i + 1
|
| 70 |
+
# Find the end - console.log line with containerInfo
|
| 71 |
+
while skip_until < len(lines) and 'container info:' not in lines[skip_until]:
|
| 72 |
+
skip_until += 1
|
| 73 |
+
skip_until += 1 # Skip the console.log line too
|
| 74 |
+
continue
|
| 75 |
+
|
| 76 |
+
# 6. Remove line chart debug block
|
| 77 |
+
if '// ── DEBUG: For line charts, verify SVG path contains all data points' in line:
|
| 78 |
+
skip_until = i + 1
|
| 79 |
+
while skip_until < len(lines) and 'LINE DEBUG:' not in lines[skip_until]:
|
| 80 |
+
skip_until += 1
|
| 81 |
+
skip_until += 1
|
| 82 |
+
continue
|
| 83 |
+
|
| 84 |
+
# 7. Remove pie chart debug block
|
| 85 |
+
if '// ── DEBUG: For pie charts, verify the actual SVG output' in line:
|
| 86 |
+
skip_until = i + 1
|
| 87 |
+
while skip_until < len(lines) and 'PIE DEBUG:' not in lines[skip_until]:
|
| 88 |
+
skip_until += 1
|
| 89 |
+
skip_until += 1
|
| 90 |
+
continue
|
| 91 |
+
|
| 92 |
+
# 8. Remove debug screenshot block
|
| 93 |
+
if '// ── DEBUG: Save screenshot to disk for visual verification' in line:
|
| 94 |
+
skip_until = i + 1
|
| 95 |
+
while skip_until < len(lines) and 'Use element screenshot' not in lines[skip_until]:
|
| 96 |
+
skip_until += 1
|
| 97 |
+
# Now skip until the dataUrl line
|
| 98 |
+
while skip_until < len(lines) and 'const dataUrl' not in lines[skip_until]:
|
| 99 |
+
skip_until += 1
|
| 100 |
+
continue
|
| 101 |
+
|
| 102 |
+
# 9. Remove PNG dimension parsing
|
| 103 |
+
if '// Parse PNG dimensions for verification' in line:
|
| 104 |
+
skip_until = i + 1
|
| 105 |
+
while skip_until < len(lines) and 'PNG dimensions:' not in lines[skip_until]:
|
| 106 |
+
skip_until += 1
|
| 107 |
+
skip_until += 1
|
| 108 |
+
continue
|
| 109 |
+
|
| 110 |
+
# 10. Remove debugId screenshot write (independent block)
|
| 111 |
+
if 'if (debugId) {' in line and i > 1050:
|
| 112 |
+
# Skip this if block and its contents
|
| 113 |
+
skip_until = i + 1
|
| 114 |
+
brace_count = 1
|
| 115 |
+
while skip_until < len(lines) and brace_count > 0:
|
| 116 |
+
if '{' in lines[skip_until]:
|
| 117 |
+
brace_count += lines[skip_until].count('{')
|
| 118 |
+
if '}' in lines[skip_until]:
|
| 119 |
+
brace_count -= lines[skip_until].count('}')
|
| 120 |
+
skip_until += 1
|
| 121 |
+
continue
|
| 122 |
+
|
| 123 |
+
new_lines.append(line)
|
| 124 |
+
|
| 125 |
+
# Verify we didn't destroy the file
|
| 126 |
+
result = ''.join(new_lines)
|
| 127 |
+
if 'buildChatGPTChartHtml' in result and 'render_charts' in result:
|
| 128 |
+
with open('D:/workAI/OpenCode/WorkSpace/backend-service/server.js', 'w', encoding='utf-8') as f:
|
| 129 |
+
f.write(result)
|
| 130 |
+
print(f"SUCCESS: Cleaned file from {len(lines)} to {len(new_lines)} lines")
|
| 131 |
+
print(f"Verified: buildChatGPTChartHtml and render_charts present")
|
| 132 |
+
else:
|
| 133 |
+
print("ERROR: Critical functions missing! NOT saving file.")
|
| 134 |
+
print(f"Result length: {len(result)} chars")
|
make/_clean_final2.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
|
| 3 |
+
with open('D:/workAI/OpenCode/WorkSpace/backend-service/server.js', 'r', encoding='utf-8') as f:
|
| 4 |
+
content = f.read()
|
| 5 |
+
|
| 6 |
+
# Remove all AXIS-DEBUG console.log lines (these are in generated HTML, run in browser)
|
| 7 |
+
# Match pattern: console.log('[AXIS-DEBUG]...');
|
| 8 |
+
# Handle both leading spaces and inline cases
|
| 9 |
+
content = re.sub(r" console\.log\('\[AXIS-DEBUG\].*?\);\n", "", content)
|
| 10 |
+
content = re.sub(r" console\.log\('\[AXIS-DEBUG\].*?\);\n", "", content)
|
| 11 |
+
content = re.sub(r" console\.log\('\[AXIS-DEBUG\].*?\);\n", "", content)
|
| 12 |
+
|
| 13 |
+
# Handle inline AXIS-DEBUG (e.g., if (x) { y; console.log('[AXIS-DEBUG]...'); })
|
| 14 |
+
content = re.sub(r"; console\.log\('\[AXIS-DEBUG\].*?'\);", "", content)
|
| 15 |
+
|
| 16 |
+
# Handle if with only AXIS-DEBUG: if (cond) console.log('[AXIS-DEBUG]...');
|
| 17 |
+
content = re.sub(r" if \(.*?\) console\.log\('\[AXIS-DEBUG\].*?\);\n", "", content)
|
| 18 |
+
|
| 19 |
+
# Remove _rawToFormatted dead code block
|
| 20 |
+
# Find the block from comment to the end of forEach
|
| 21 |
+
pattern = r" // .*?Build format map from chartData.*?\n var _rawToFormatted = \{\};.*?}\n\}\);\n\n // Build series elements"
|
| 22 |
+
content = re.sub(pattern, " // Build series elements", content, flags=re.DOTALL)
|
| 23 |
+
|
| 24 |
+
# Remove verbose PIE DATA logging
|
| 25 |
+
content = content.replace(
|
| 26 |
+
" if (chartType === 'pie' || chartType === 'donut') {\n console.log(`[WIDGET] renderChatGPTChart PIE DATA: data=${JSON.stringify(data)}, formattedData=${JSON.stringify(formattedData)}`);\n }",
|
| 27 |
+
""
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
# Remove verbose nameKey/valueKey logging
|
| 31 |
+
content = content.replace(
|
| 32 |
+
" if (nameKey) console.log(`[WIDGET] renderChatGPTChart: nameKey=${nameKey}, valueKey=${valueKey}`);",
|
| 33 |
+
""
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
# Remove duplicate COLORS
|
| 37 |
+
old_colors = """ const COLORS = [
|
| 38 |
+
'#339CFF', '#40C977', '#FF8549', '#FFD240',
|
| 39 |
+
'#339CFF', '#40C977', '#FF8549', '#FFD240',
|
| 40 |
+
'#339CFF', '#40C977', '#FF8549', '#FFD240',
|
| 41 |
+
];"""
|
| 42 |
+
new_colors = """ const COLORS = ['#339CFF', '#40C977', '#FF8549', '#FFD240'];"""
|
| 43 |
+
if old_colors in content:
|
| 44 |
+
content = content.replace(old_colors, new_colors)
|
| 45 |
+
|
| 46 |
+
# Remove dead baseWaitTime/sizeWaitTime/totalWaitTime
|
| 47 |
+
content = re.sub(r"\n const baseWaitTime = imgCount > 0.*?\n const sizeWaitTime = imgSizeMB.*?\n const totalWaitTime = Math\.min.*?\n", "\n", content)
|
| 48 |
+
|
| 49 |
+
# Remove redundant %%INJECTED_SCRIPTS%% placeholder
|
| 50 |
+
content = content.replace(".replace('%%INJECTED_SCRIPTS%%', '');", ";")
|
| 51 |
+
|
| 52 |
+
# Remove dead pieActualDataKey
|
| 53 |
+
content = content.replace("var pieActualDataKey = valueKey || chartSeries[0]?.dataKey || 'value';\n", "")
|
| 54 |
+
|
| 55 |
+
# Verify critical functions still present
|
| 56 |
+
if 'buildChatGPTChartHtml' in content and 'render_charts' in content and 'isAnimationActive: false' in content:
|
| 57 |
+
with open('D:/workAI/OpenCode/WorkSpace/backend-service/server.js', 'w', encoding='utf-8') as f:
|
| 58 |
+
f.write(content)
|
| 59 |
+
print("SUCCESS: All cleanup applied")
|
| 60 |
+
else:
|
| 61 |
+
print("ERROR: Critical content missing! NOT saving.")
|
make/backup-source.js
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
const fs = require('fs');
|
| 2 |
+
const path = require('path');
|
| 3 |
+
|
| 4 |
+
const BACKUP_BASE = path.resolve(__dirname, '..', '..');
|
| 5 |
+
const SRC_DIR = path.resolve(__dirname, '..');
|
| 6 |
+
|
| 7 |
+
const EXCLUDE_DIRS = ['node_modules', '.git', '.claude', 'dist'];
|
| 8 |
+
|
| 9 |
+
function shouldExclude(full, srcDir) {
|
| 10 |
+
const relative = path.relative(srcDir, full).replace(/\\/g, '/');
|
| 11 |
+
const basename = path.basename(full);
|
| 12 |
+
for (const d of EXCLUDE_DIRS) {
|
| 13 |
+
if (relative === d || relative.startsWith(d + '/')) return true;
|
| 14 |
+
}
|
| 15 |
+
return false;
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
function collectFiles(srcDir) {
|
| 19 |
+
const files = [];
|
| 20 |
+
function walk(dir) {
|
| 21 |
+
const entries = fs.readdirSync(dir);
|
| 22 |
+
for (const entry of entries) {
|
| 23 |
+
const full = path.join(dir, entry);
|
| 24 |
+
if (shouldExclude(full, srcDir)) continue;
|
| 25 |
+
const stat = fs.statSync(full);
|
| 26 |
+
if (stat.isDirectory()) walk(full);
|
| 27 |
+
else files.push(full);
|
| 28 |
+
}
|
| 29 |
+
}
|
| 30 |
+
walk(srcDir);
|
| 31 |
+
return files;
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
function copyFiles(files, srcDir, destDir) {
|
| 35 |
+
for (const full of files) {
|
| 36 |
+
const relative = path.relative(srcDir, full);
|
| 37 |
+
const dest = path.join(destDir, relative);
|
| 38 |
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
| 39 |
+
fs.copyFileSync(full, dest);
|
| 40 |
+
}
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
const pkgPath = path.join(SRC_DIR, 'package.json');
|
| 44 |
+
const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8'));
|
| 45 |
+
const version = pkg.version || 'unknown';
|
| 46 |
+
|
| 47 |
+
const now = new Date();
|
| 48 |
+
const ts = now.toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
| 49 |
+
|
| 50 |
+
const folderName = 'backend-service_v' + version + '_' + ts;
|
| 51 |
+
const backupPath = path.join(BACKUP_BASE, folderName);
|
| 52 |
+
|
| 53 |
+
console.log('Version: ' + version);
|
| 54 |
+
console.log('Creating backup: ' + backupPath);
|
| 55 |
+
|
| 56 |
+
const files = collectFiles(SRC_DIR);
|
| 57 |
+
console.log('Found ' + files.length + ' source files');
|
| 58 |
+
|
| 59 |
+
copyFiles(files, SRC_DIR, backupPath);
|
| 60 |
+
console.log('Backup complete: ' + backupPath);
|
| 61 |
+
|
| 62 |
+
const serverJs = fs.readFileSync(path.join(backupPath, 'server.js'), 'utf-8');
|
| 63 |
+
const hasNewColors = serverJs.includes('#339CFF');
|
| 64 |
+
console.log('New ChatGPT colors in backup: ' + (hasNewColors ? 'YES' : 'NO'));
|
package-lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
| 1 |
{
|
| 2 |
"name": "pdf-server",
|
| 3 |
-
"version": "2.
|
| 4 |
"lockfileVersion": 3,
|
| 5 |
"requires": true,
|
| 6 |
"packages": {
|
| 7 |
"": {
|
| 8 |
"name": "pdf-server",
|
| 9 |
-
"version": "2.
|
| 10 |
"dependencies": {
|
| 11 |
"@mermaid-js/mermaid-cli": "^11.0.0",
|
| 12 |
"chart.js": "^4.4.8",
|
|
|
|
| 1 |
{
|
| 2 |
"name": "pdf-server",
|
| 3 |
+
"version": "2.1.7",
|
| 4 |
"lockfileVersion": 3,
|
| 5 |
"requires": true,
|
| 6 |
"packages": {
|
| 7 |
"": {
|
| 8 |
"name": "pdf-server",
|
| 9 |
+
"version": "2.1.7",
|
| 10 |
"dependencies": {
|
| 11 |
"@mermaid-js/mermaid-cli": "^11.0.0",
|
| 12 |
"chart.js": "^4.4.8",
|
package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "pdf-server",
|
| 3 |
-
"version": "2.
|
| 4 |
"description": "Puppeteer PDF + Widget Renderer for XWX AI Chat Exporter",
|
| 5 |
"main": "server.js",
|
| 6 |
"dependencies": {
|
|
|
|
| 1 |
{
|
| 2 |
"name": "pdf-server",
|
| 3 |
+
"version": "2.1.11",
|
| 4 |
"description": "Puppeteer PDF + Widget Renderer for XWX AI Chat Exporter",
|
| 5 |
"main": "server.js",
|
| 6 |
"dependencies": {
|
server.js
CHANGED
|
@@ -29,6 +29,7 @@ const { getHighlighter } = require('shiki');
|
|
| 29 |
const fs = require('fs');
|
| 30 |
const path = require('path');
|
| 31 |
const os = require('os');
|
|
|
|
| 32 |
|
| 33 |
let ChartJSNodeCanvas = null;
|
| 34 |
try {
|
|
@@ -122,6 +123,99 @@ const isTest = process.env.NODE_ENV === 'test';
|
|
| 122 |
|
| 123 |
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
| 124 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
app.use(cors());
|
| 126 |
app.use(express.json({ limit: '50mb' }));
|
| 127 |
|
|
@@ -150,6 +244,9 @@ app.post('/api/generate_pdf', async (req, res) => {
|
|
| 150 |
|
| 151 |
const getElapsed = () => ((Date.now() - startTime) / 1000).toFixed(2) + 's';
|
| 152 |
let browser = null;
|
|
|
|
|
|
|
|
|
|
| 153 |
|
| 154 |
try {
|
| 155 |
if (!html) {
|
|
@@ -262,27 +359,21 @@ app.post('/api/generate_pdf', async (req, res) => {
|
|
| 262 |
const estMinutes = (pdfTimeout / 60000).toFixed(0);
|
| 263 |
console.log(`[PDF-GEN] [${getElapsed()}] ⚠️ 大文件预警: 纯文本=${effectiveSizeMB.toFixed(2)} MB HTML 预计需要 ${estMinutes} 分钟`);
|
| 264 |
}
|
| 265 |
-
console.log(`[PDF-GEN] [${getElapsed()}] 正在
|
| 266 |
// protocolTimeout: 0 = 禁用 CDP 协议层超时
|
| 267 |
// 参考: https://github.com/puppeteer/puppeteer/issues/9927
|
| 268 |
// PDF 超时时由应用层 Promise.race 控制,不依赖协议层超时
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
'--disable-software-rasterizer',
|
| 279 |
-
'--memory-pressure-off'
|
| 280 |
-
],
|
| 281 |
-
headless: 'shell'
|
| 282 |
});
|
| 283 |
-
console.log(`[PDF-GEN] [${getElapsed()}] 浏览器启动成功`);
|
| 284 |
|
| 285 |
-
const page = await browser.newPage();
|
| 286 |
// 设置 viewport 满足大部分页面渲染需求
|
| 287 |
await page.setViewport({ width: 1200, height: 800 });
|
| 288 |
console.log(`[PDF-GEN] [${getElapsed()}] Viewport: 1200x800`);
|
|
@@ -290,25 +381,34 @@ app.post('/api/generate_pdf', async (req, res) => {
|
|
| 290 |
// 大 HTML (> 5 MB) 使用临时文件法,避免 CDP WebSocket 传输限制
|
| 291 |
// 参考: https://danindu.medium.com/optimizing-puppeteer-for-pdf-generation-8b7777edbeca
|
| 292 |
const isLargeHtml = htmlSizeMBNum > 5;
|
| 293 |
-
let tempFilePath = null;
|
| 294 |
|
| 295 |
console.log(`[PDF-GEN] [${getElapsed()}] 正在${isLargeHtml ? '通过临时文件加载' : '填充'}页面内容...`);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 296 |
if (isLargeHtml) {
|
| 297 |
tempFilePath = path.join(os.tmpdir(), `xwx-pdf-${Date.now()}.html`);
|
| 298 |
-
fs.
|
| 299 |
await page.goto(`file://${tempFilePath}`, {
|
| 300 |
-
waitUntil:
|
| 301 |
timeout: setContentTimeout
|
| 302 |
});
|
| 303 |
} else {
|
| 304 |
await page.setContent(htmlToUse, {
|
| 305 |
-
waitUntil:
|
| 306 |
timeout: setContentTimeout
|
| 307 |
});
|
| 308 |
}
|
| 309 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
console.log(`[PDF-GEN] [${getElapsed()}] 页面内容加载完成`);
|
| 311 |
-
|
| 312 |
// 等待 base64 图片完全渲染(检测实际加载状态)
|
| 313 |
let loadedImages = null;
|
| 314 |
if (imgCount > 0) {
|
|
@@ -333,16 +433,25 @@ app.post('/api/generate_pdf', async (req, res) => {
|
|
| 333 |
results.push({ src: srcPreview, status, width: img.naturalWidth, isBase64 });
|
| 334 |
}
|
| 335 |
|
| 336 |
-
//
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
const base64Images = Array.from(images).filter(img => {
|
| 338 |
const src = img.getAttribute('src') || '';
|
| 339 |
return src.startsWith('data:image/') && src.length > 100;
|
| 340 |
});
|
| 341 |
-
|
| 342 |
if (base64Images.length > 0) {
|
| 343 |
-
console.log(` 检测到 ${base64Images.length} 张base64图片,跳过onload等待
|
| 344 |
-
}
|
| 345 |
-
|
|
|
|
|
|
|
| 346 |
if (img.complete && img.naturalWidth > 0) {
|
| 347 |
return Promise.resolve();
|
| 348 |
}
|
|
@@ -459,12 +568,12 @@ app.post('/api/generate_pdf', async (req, res) => {
|
|
| 459 |
|
| 460 |
// 清理临时文件
|
| 461 |
if (tempFilePath) {
|
| 462 |
-
try { fs.
|
| 463 |
console.log(`[PDF-GEN] [${getElapsed()}] 已清理临时文件`);
|
| 464 |
}
|
| 465 |
|
| 466 |
const pdfSizeMB = (pdfBuffer.length / 1024 / 1024).toFixed(2);
|
| 467 |
-
console.log(`[PDF-GEN] [${getElapsed()}] PDF 生成成功 (${pdfSizeMB} MB),正在
|
| 468 |
|
| 469 |
// Pass image loading summary as response header for frontend debugging
|
| 470 |
if (loadedImages && loadedImages.final) {
|
|
@@ -473,23 +582,49 @@ app.post('/api/generate_pdf', async (req, res) => {
|
|
| 473 |
res.setHeader('X-Image-Count', String(loadedImages.final.length));
|
| 474 |
}
|
| 475 |
|
| 476 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 477 |
browser = null;
|
| 478 |
|
| 479 |
console.log(`[PDF-GEN] [${getElapsed()}] >>> 任务全部完成 <<<`);
|
| 480 |
|
| 481 |
res.setHeader('Content-Type', 'application/pdf');
|
| 482 |
res.setHeader('Content-Disposition', 'attachment; filename=export.pdf');
|
|
|
|
|
|
|
|
|
|
| 483 |
res.send(Buffer.from(pdfBuffer));
|
| 484 |
|
| 485 |
} catch (error) {
|
| 486 |
console.error(`[PDF-GEN] [${getElapsed()}] 发生错误:`, error);
|
| 487 |
-
|
| 488 |
-
|
|
|
|
|
|
|
| 489 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 490 |
// 清理临时文件
|
| 491 |
if (tempFilePath) {
|
| 492 |
-
try { fs.
|
| 493 |
}
|
| 494 |
res.status(500).json({ error: 'Internal Server Error', details: error.message });
|
| 495 |
}
|
|
@@ -500,37 +635,15 @@ app.post('/api/generate_pdf', async (req, res) => {
|
|
| 500 |
class WidgetRenderer {
|
| 501 |
constructor() {
|
| 502 |
this._chartInstances = new Map();
|
| 503 |
-
this._widgetBrowser = null;
|
| 504 |
-
}
|
| 505 |
-
|
| 506 |
-
async getWidgetBrowser() {
|
| 507 |
-
if (this._widgetBrowser && this._widgetBrowser.isConnected()) {
|
| 508 |
-
return this._widgetBrowser;
|
| 509 |
-
}
|
| 510 |
-
this._widgetBrowser = await puppeteer.launch({
|
| 511 |
-
executablePath: '/usr/bin/chromium',
|
| 512 |
-
args: [
|
| 513 |
-
'--no-sandbox',
|
| 514 |
-
'--disable-setuid-sandbox',
|
| 515 |
-
'--disable-dev-shm-usage',
|
| 516 |
-
'--font-render-hinting=none',
|
| 517 |
-
'--disable-gpu',
|
| 518 |
-
'--disable-software-rasterizer',
|
| 519 |
-
'--enable-webgl',
|
| 520 |
-
'--use-gl=angle',
|
| 521 |
-
'--use-angle=swiftshader',
|
| 522 |
-
'--memory-pressure-off'
|
| 523 |
-
],
|
| 524 |
-
headless: 'shell'
|
| 525 |
-
});
|
| 526 |
-
return this._widgetBrowser;
|
| 527 |
}
|
| 528 |
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
|
|
|
|
|
|
| 534 |
}
|
| 535 |
|
| 536 |
_getChartInstance(width, height) {
|
|
@@ -842,34 +955,48 @@ class WidgetRenderer {
|
|
| 842 |
* The HTML already contains full <html>/<head>/<body> with script tags.
|
| 843 |
*/
|
| 844 |
async _renderFullHtml(htmlContent, title, renderTimeout, chartType) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 845 |
const startTime = Date.now();
|
| 846 |
console.log(`[WIDGET] _renderFullHtml START: title=${title}, timeout=${renderTimeout}ms, chartType=${chartType}`);
|
| 847 |
|
| 848 |
-
let
|
|
|
|
| 849 |
try {
|
| 850 |
-
|
| 851 |
-
|
| 852 |
-
|
| 853 |
-
|
| 854 |
-
'--disable-gpu',
|
| 855 |
-
'--disable-software-rasterizer',
|
| 856 |
-
'--memory-pressure-off'
|
| 857 |
-
];
|
| 858 |
-
|
| 859 |
-
browser = await puppeteer.launch({
|
| 860 |
-
executablePath: '/usr/bin/chromium',
|
| 861 |
-
args: launchArgs,
|
| 862 |
-
headless: 'shell'
|
| 863 |
-
});
|
| 864 |
-
|
| 865 |
-
const page = await browser.newPage();
|
| 866 |
|
| 867 |
// CRITICAL: Use a compact viewport height to avoid excessive whitespace
|
| 868 |
// in fullPage screenshots. Width 800 gives enough room for 650px chart.
|
| 869 |
await page.setViewport({ width: 800, height: 500 });
|
| 870 |
|
| 871 |
// Set complete HTML document directly
|
| 872 |
-
|
|
|
|
|
|
|
| 873 |
|
| 874 |
// Wait for React/Recharts to render (check window._widgetRendered flag)
|
| 875 |
const rendered = await page.waitForFunction(
|
|
@@ -900,8 +1027,12 @@ class WidgetRenderer {
|
|
| 900 |
console.log(`[WIDGET] _renderFullHtml ERROR: ${e.message}`);
|
| 901 |
return null;
|
| 902 |
} finally {
|
| 903 |
-
|
| 904 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 905 |
}
|
| 906 |
}
|
| 907 |
}
|
|
@@ -1316,29 +1447,47 @@ window._widgetRendered = false;
|
|
| 1316 |
}
|
| 1317 |
|
| 1318 |
async renderWidgetPuppeteer(widgetHtml, type, title, renderTimeout) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1319 |
const startTime = Date.now();
|
| 1320 |
console.log(`[WIDGET] renderPuppeteer START: type=${type}, title=${title}, timeout=${renderTimeout}ms`);
|
| 1321 |
|
| 1322 |
-
let
|
|
|
|
| 1323 |
try {
|
| 1324 |
const finalTimeout = renderTimeout || 3000;
|
| 1325 |
|
| 1326 |
-
|
| 1327 |
-
|
| 1328 |
-
|
| 1329 |
-
|
| 1330 |
-
'--disable-gpu',
|
| 1331 |
-
'--disable-software-rasterizer',
|
| 1332 |
-
'--memory-pressure-off'
|
| 1333 |
-
];
|
| 1334 |
-
|
| 1335 |
-
browser = await puppeteer.launch({
|
| 1336 |
-
executablePath: '/usr/bin/chromium',
|
| 1337 |
-
args: launchArgs,
|
| 1338 |
-
headless: 'shell'
|
| 1339 |
-
});
|
| 1340 |
|
| 1341 |
-
|
| 1342 |
await page.setViewport({ width: 1024, height: 768 });
|
| 1343 |
|
| 1344 |
const templatesDir = path.join(__dirname, 'templates');
|
|
@@ -1374,9 +1523,17 @@ window._widgetRendered = false;
|
|
| 1374 |
let fullHtml = template
|
| 1375 |
.replace('%%WIDGET_CODE%%', widgetCode);
|
| 1376 |
|
| 1377 |
-
|
|
|
|
|
|
|
| 1378 |
|
| 1379 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1380 |
|
| 1381 |
let renderComplete = false;
|
| 1382 |
|
|
@@ -1512,8 +1669,12 @@ window._widgetRendered = false;
|
|
| 1512 |
console.log(`[WIDGET] renderPuppeteer ERROR: ${e.message}`);
|
| 1513 |
return null;
|
| 1514 |
} finally {
|
| 1515 |
-
|
| 1516 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1517 |
}
|
| 1518 |
}
|
| 1519 |
}
|
|
@@ -1541,26 +1702,47 @@ window._widgetRendered = false;
|
|
| 1541 |
}
|
| 1542 |
|
| 1543 |
const widgetRenderer = new WidgetRenderer();
|
| 1544 |
-
const MAX_CONCURRENT_RENDER =
|
| 1545 |
let activeRenderCount = 0;
|
| 1546 |
const renderQueue = [];
|
| 1547 |
|
| 1548 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1549 |
return new Promise((resolve, reject) => {
|
| 1550 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1551 |
processRenderQueue();
|
| 1552 |
});
|
| 1553 |
}
|
| 1554 |
|
| 1555 |
async function processRenderQueue() {
|
| 1556 |
while (renderQueue.length > 0 && activeRenderCount < MAX_CONCURRENT_RENDER) {
|
| 1557 |
-
const
|
|
|
|
| 1558 |
activeRenderCount++;
|
| 1559 |
try {
|
| 1560 |
-
const result = await widgetRenderer.render(widget);
|
| 1561 |
-
resolve(result);
|
| 1562 |
} catch (e) {
|
| 1563 |
-
reject(e);
|
| 1564 |
} finally {
|
| 1565 |
activeRenderCount--;
|
| 1566 |
processRenderQueue();
|
|
@@ -1591,10 +1773,11 @@ app.post('/api/render_charts', async (req, res) => {
|
|
| 1591 |
console.log(`[WIDGET] [${index + 1}/${widgets.length}] Rendering: type=${wType}, title=${wTitle}`);
|
| 1592 |
|
| 1593 |
try {
|
| 1594 |
-
const renderPromise = renderWithConcurrency(widget);
|
| 1595 |
// Per-widget timeout: mermaid needs extra time for CDN loading + rendering (~15s).
|
| 1596 |
// Other widgets complete in <5s. Using a generous 60s timeout for all types.
|
| 1597 |
const widgetTimeout = wType === 'mermaid' ? 60000 : 30000;
|
|
|
|
|
|
|
| 1598 |
const timeoutPromise = new Promise(resolve => setTimeout(() => resolve(null), widgetTimeout));
|
| 1599 |
const dataUrl = await Promise.race([renderPromise, timeoutPromise]);
|
| 1600 |
|
|
@@ -1623,6 +1806,18 @@ app.post('/api/render_charts', async (req, res) => {
|
|
| 1623 |
res.json({ results: renderResults });
|
| 1624 |
});
|
| 1625 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1626 |
app.listen(port, () => {
|
| 1627 |
console.log(`Server listening at http://localhost:${port}`);
|
| 1628 |
});
|
|
|
|
| 29 |
const fs = require('fs');
|
| 30 |
const path = require('path');
|
| 31 |
const os = require('os');
|
| 32 |
+
const { BrowserPool } = require('./browser-pool');
|
| 33 |
|
| 34 |
let ChartJSNodeCanvas = null;
|
| 35 |
try {
|
|
|
|
| 123 |
|
| 124 |
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
| 125 |
|
| 126 |
+
// ─── Performance configuration (env overridable) ────────────────────
|
| 127 |
+
// PDF_POOL_SIZE: how many Chromium browsers to keep warm. Each browser
|
| 128 |
+
// renders ONE PDF at a time (page.pdf() is CPU-bound), so this is also the
|
| 129 |
+
// max concurrent PDF jobs. Default 4 works well on multi-core; set to 2 for
|
| 130 |
+
// Hugging Face free tier (2 vCPU / 16GB).
|
| 131 |
+
// PDF_RECYCLE_AFTER: recycle a browser after N jobs to bound memory growth.
|
| 132 |
+
// WIDGET_MAX_CONCURRENT: concurrent widget renders inside the shared widget
|
| 133 |
+
// browser (each widget gets its own page).
|
| 134 |
+
const PDF_POOL_SIZE = parseInt(process.env.PDF_POOL_SIZE || '4', 10);
|
| 135 |
+
const PDF_RECYCLE_AFTER = parseInt(process.env.PDF_RECYCLE_AFTER || '30', 10);
|
| 136 |
+
const PDF_ACQUIRE_TIMEOUT_MS = parseInt(process.env.PDF_ACQUIRE_TIMEOUT_MS || '120000', 10);
|
| 137 |
+
const WIDGET_MAX_CONCURRENT = parseInt(process.env.WIDGET_MAX_CONCURRENT || '3', 10);
|
| 138 |
+
// 单次 CDP 调用(page.evaluate / screenshot / setContent 等)的最大等待时间。
|
| 139 |
+
// 必须小于 render_charts 的 30s 应用级超时,确保挂起操作能及时释放并发槽位。
|
| 140 |
+
const WIDGET_CDP_TIMEOUT_MS = parseInt(process.env.WIDGET_CDP_TIMEOUT_MS || '20000', 10);
|
| 141 |
+
// 单个 widget 整个渲染流程的硬超时(无论内部卡在哪一步),保证渲染必然在
|
| 142 |
+
// 应用级 30s 超时之前结束,并发槽位不会被无限期占用。
|
| 143 |
+
const WIDGET_HARD_TIMEOUT_MS = parseInt(process.env.WIDGET_HARD_TIMEOUT_MS || '20000', 10);
|
| 144 |
+
// 从 widget 浏览器池获取浏览器的最长等待时间(池满时排队)
|
| 145 |
+
const WIDGET_ACQUIRE_TIMEOUT_MS = parseInt(process.env.WIDGET_ACQUIRE_TIMEOUT_MS || '15000', 10);
|
| 146 |
+
|
| 147 |
+
const PDF_LAUNCH_ARGS = [
|
| 148 |
+
'--no-sandbox',
|
| 149 |
+
'--disable-setuid-sandbox',
|
| 150 |
+
'--disable-dev-shm-usage',
|
| 151 |
+
'--font-render-hinting=none',
|
| 152 |
+
'--disable-gpu',
|
| 153 |
+
'--disable-software-rasterizer',
|
| 154 |
+
'--memory-pressure-off'
|
| 155 |
+
];
|
| 156 |
+
|
| 157 |
+
const PDF_LAUNCH_OPTIONS = {
|
| 158 |
+
executablePath: '/usr/bin/chromium',
|
| 159 |
+
protocolTimeout: 0,
|
| 160 |
+
// Load images served with expired/invalid SSL certificates.
|
| 161 |
+
// Third-party image CDNs (e.g. imgs.sbkko.com) can have certificate issues;
|
| 162 |
+
// the PDF must still render those images the user saw in the conversation.
|
| 163 |
+
acceptInsecureCerts: true,
|
| 164 |
+
args: PDF_LAUNCH_ARGS,
|
| 165 |
+
headless: 'shell'
|
| 166 |
+
};
|
| 167 |
+
|
| 168 |
+
const pdfPool = new BrowserPool({
|
| 169 |
+
name: 'pdf',
|
| 170 |
+
size: PDF_POOL_SIZE,
|
| 171 |
+
launchOptions: PDF_LAUNCH_OPTIONS,
|
| 172 |
+
recycleAfter: PDF_RECYCLE_AFTER,
|
| 173 |
+
acquireTimeoutMs: PDF_ACQUIRE_TIMEOUT_MS,
|
| 174 |
+
log: (msg) => console.log(`[PERF] ${msg}`)
|
| 175 |
+
});
|
| 176 |
+
|
| 177 |
+
// ─── Widget browser pool ───────────────────────────────────────
|
| 178 |
+
// 为什么用「池」而不是单例浏览器(v2.1.7 曾改为单例 + 高并发 page):
|
| 179 |
+
// 单例浏览器下多个 widget 同时渲染会争抢同一个浏览器进程的页面主线程
|
| 180 |
+
// (Runtime.evaluate / screenshot 实测可挂起 20s+,最严重 231s),单个
|
| 181 |
+
// widget 拖慢后占满全局并发队列,导致后续 widget 排队直至 30s 超时级联,
|
| 182 |
+
// DOCX 导出因此把图表 fallback 成数据表格。池化后每个 widget 独占一个
|
| 183 |
+
// 浏览器(进程级隔离,等同稳定版 v2.0.8 的每 widget 一浏览器),同时保留
|
| 184 |
+
// 浏览器复用(避免每次启动 Chromium 的开销)。池大小 = WIDGET_MAX_CONCURRENT。
|
| 185 |
+
const WIDGET_LAUNCH_OPTIONS = {
|
| 186 |
+
executablePath: '/usr/bin/chromium',
|
| 187 |
+
// 有限 CDP 超时(而非 PDF 的 0):池内浏览器被 widget 共享复用,单次 CDP
|
| 188 |
+
// 调用挂起必须在应用级 30s 超时之前抛错并释放浏览器,否则会拖垮整个池。
|
| 189 |
+
protocolTimeout: WIDGET_CDP_TIMEOUT_MS,
|
| 190 |
+
// Same rationale as PDF_LAUNCH_OPTIONS: widget HTML may embed images
|
| 191 |
+
// from third-party CDNs with expired/invalid certificates.
|
| 192 |
+
acceptInsecureCerts: true,
|
| 193 |
+
args: [
|
| 194 |
+
'--no-sandbox',
|
| 195 |
+
'--disable-setuid-sandbox',
|
| 196 |
+
'--disable-dev-shm-usage',
|
| 197 |
+
'--font-render-hinting=none',
|
| 198 |
+
'--disable-gpu',
|
| 199 |
+
'--disable-software-rasterizer',
|
| 200 |
+
'--enable-webgl',
|
| 201 |
+
'--use-gl=angle',
|
| 202 |
+
'--use-angle=swiftshader',
|
| 203 |
+
'--memory-pressure-off'
|
| 204 |
+
],
|
| 205 |
+
headless: 'shell'
|
| 206 |
+
};
|
| 207 |
+
|
| 208 |
+
const widgetPool = new BrowserPool({
|
| 209 |
+
name: 'widget',
|
| 210 |
+
size: WIDGET_MAX_CONCURRENT,
|
| 211 |
+
launchOptions: WIDGET_LAUNCH_OPTIONS,
|
| 212 |
+
recycleAfter: 100,
|
| 213 |
+
acquireTimeoutMs: WIDGET_ACQUIRE_TIMEOUT_MS,
|
| 214 |
+
log: (msg) => console.log(`[PERF] ${msg}`)
|
| 215 |
+
});
|
| 216 |
+
|
| 217 |
+
console.log(`[PERF] PDF browser pool: size=${PDF_POOL_SIZE}, recycleAfter=${PDF_RECYCLE_AFTER}, widgetConcurrent=${WIDGET_MAX_CONCURRENT}, widgetPoolSize=${WIDGET_MAX_CONCURRENT}`);
|
| 218 |
+
|
| 219 |
app.use(cors());
|
| 220 |
app.use(express.json({ limit: '50mb' }));
|
| 221 |
|
|
|
|
| 244 |
|
| 245 |
const getElapsed = () => ((Date.now() - startTime) / 1000).toFixed(2) + 's';
|
| 246 |
let browser = null;
|
| 247 |
+
let acquired = null; // pooled browser slot (release in finally)
|
| 248 |
+
let page = null;
|
| 249 |
+
let tempFilePath = null;
|
| 250 |
|
| 251 |
try {
|
| 252 |
if (!html) {
|
|
|
|
| 359 |
const estMinutes = (pdfTimeout / 60000).toFixed(0);
|
| 360 |
console.log(`[PDF-GEN] [${getElapsed()}] ⚠️ 大文件预警: 纯文本=${effectiveSizeMB.toFixed(2)} MB HTML 预计需要 ${estMinutes} 分钟`);
|
| 361 |
}
|
| 362 |
+
console.log(`[PDF-GEN] [${getElapsed()}] 正在从浏览器池获取浏览器 (池大小=${PDF_POOL_SIZE})...`);
|
| 363 |
// protocolTimeout: 0 = 禁用 CDP 协议层超时
|
| 364 |
// 参考: https://github.com/puppeteer/puppeteer/issues/9927
|
| 365 |
// PDF 超时时由应用层 Promise.race 控制,不依赖协议层超时
|
| 366 |
+
acquired = await pdfPool.acquire();
|
| 367 |
+
browser = acquired.browser;
|
| 368 |
+
page = await browser.newPage();
|
| 369 |
+
|
| 370 |
+
// 兜底:自动关闭 alert/confirm/prompt 对话框,防止 XSS/异常 HTML 卡死 setContent
|
| 371 |
+
// 参考解决方案备忘录 24 (iframe javascript: URL + dialog = Puppeteer 永久阻塞)
|
| 372 |
+
page.on('dialog', async dialog => {
|
| 373 |
+
console.warn(`[PDF-GEN] [${getElapsed()}] Auto-dismissed ${dialog.type()} dialog: ${dialog.message().substring(0, 100)}`);
|
| 374 |
+
try { await dialog.dismiss(); } catch (e) {}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 375 |
});
|
|
|
|
| 376 |
|
|
|
|
| 377 |
// 设置 viewport 满足大部分页面渲染需求
|
| 378 |
await page.setViewport({ width: 1200, height: 800 });
|
| 379 |
console.log(`[PDF-GEN] [${getElapsed()}] Viewport: 1200x800`);
|
|
|
|
| 381 |
// 大 HTML (> 5 MB) 使用临时文件法,避免 CDP WebSocket 传输限制
|
| 382 |
// 参考: https://danindu.medium.com/optimizing-puppeteer-for-pdf-generation-8b7777edbeca
|
| 383 |
const isLargeHtml = htmlSizeMBNum > 5;
|
|
|
|
| 384 |
|
| 385 |
console.log(`[PDF-GEN] [${getElapsed()}] 正在${isLargeHtml ? '通过临时文件加载' : '填充'}页面内容...`);
|
| 386 |
+
// waitUntil 策略:用 'domcontentloaded' 而非 'load'。
|
| 387 |
+
// 原因:'load' 会等待页面所有子资源(含外部图片)加载完成。若某张外部图片
|
| 388 |
+
// 的 CDN 响应挂起(既不成功也不失败,例如证书/网络异常),load 事件永不触发,
|
| 389 |
+
// 导致整个 PDF 请求超时失败。改为 domcontentloaded 后 DOM 就绪即继续,
|
| 390 |
+
// 图片由下方的 per-image 有界等待逻辑兜底,单张慢图不会阻塞整个 PDF。
|
| 391 |
if (isLargeHtml) {
|
| 392 |
tempFilePath = path.join(os.tmpdir(), `xwx-pdf-${Date.now()}.html`);
|
| 393 |
+
await fs.promises.writeFile(tempFilePath, htmlToUse, 'utf8');
|
| 394 |
await page.goto(`file://${tempFilePath}`, {
|
| 395 |
+
waitUntil: 'domcontentloaded',
|
| 396 |
timeout: setContentTimeout
|
| 397 |
});
|
| 398 |
} else {
|
| 399 |
await page.setContent(htmlToUse, {
|
| 400 |
+
waitUntil: 'domcontentloaded',
|
| 401 |
timeout: setContentTimeout
|
| 402 |
});
|
| 403 |
}
|
| 404 |
+
// 有界等待网络空闲。图片加载状态由下方 per-image 超时逻辑兜底;
|
| 405 |
+
// 此处超时仅告警不中断,防止挂起的外部资源阻塞整个 PDF 生成。
|
| 406 |
+
try {
|
| 407 |
+
await page.waitForNetworkIdle({ idleTime: 300, timeout: Math.min(networkTimeout, 15000) });
|
| 408 |
+
} catch (e) {
|
| 409 |
+
console.log(`[PDF-GEN] [${getElapsed()}] waitForNetworkIdle 超时(非致命,图片检测将继续): ${e.message}`);
|
| 410 |
+
}
|
| 411 |
console.log(`[PDF-GEN] [${getElapsed()}] 页面内容加载完成`);
|
|
|
|
| 412 |
// 等待 base64 图片完全渲染(检测实际加载状态)
|
| 413 |
let loadedImages = null;
|
| 414 |
if (imgCount > 0) {
|
|
|
|
| 433 |
results.push({ src: srcPreview, status, width: img.naturalWidth, isBase64 });
|
| 434 |
}
|
| 435 |
|
| 436 |
+
// 外部(非 base64)图片可能有界等待加载完成:
|
| 437 |
+
// - base64 图片随 DOM 同步解码,无需等待(headless shell 模式 complete 也不准确)
|
| 438 |
+
// - 外部图片受 CDN 网络影响,统一等待 onload/onerror,最多 15s 兜底
|
| 439 |
+
// - 这样单���挂起的外部图片不会阻塞整个 PDF(与 waitUntil:'domcontentloaded' 配合)
|
| 440 |
+
const externalImages = Array.from(images).filter(img => {
|
| 441 |
+
const src = img.getAttribute('src') || '';
|
| 442 |
+
return !src.startsWith('data:image/');
|
| 443 |
+
});
|
| 444 |
const base64Images = Array.from(images).filter(img => {
|
| 445 |
const src = img.getAttribute('src') || '';
|
| 446 |
return src.startsWith('data:image/') && src.length > 100;
|
| 447 |
});
|
| 448 |
+
|
| 449 |
if (base64Images.length > 0) {
|
| 450 |
+
console.log(` 检测到 ${base64Images.length} 张base64图片(同步解码,跳过onload等待)`);
|
| 451 |
+
}
|
| 452 |
+
if (externalImages.length > 0) {
|
| 453 |
+
console.log(` 等待 ${externalImages.length} 张外部图片加载(每张最多 15s 兜底)...`);
|
| 454 |
+
await Promise.all(externalImages.map(img => {
|
| 455 |
if (img.complete && img.naturalWidth > 0) {
|
| 456 |
return Promise.resolve();
|
| 457 |
}
|
|
|
|
| 568 |
|
| 569 |
// 清理临时文件
|
| 570 |
if (tempFilePath) {
|
| 571 |
+
try { await fs.promises.unlink(tempFilePath); } catch (e) {}
|
| 572 |
console.log(`[PDF-GEN] [${getElapsed()}] 已清理临时文件`);
|
| 573 |
}
|
| 574 |
|
| 575 |
const pdfSizeMB = (pdfBuffer.length / 1024 / 1024).toFixed(2);
|
| 576 |
+
console.log(`[PDF-GEN] [${getElapsed()}] PDF 生成成功 (${pdfSizeMB} MB),正在释放浏览器回池...`);
|
| 577 |
|
| 578 |
// Pass image loading summary as response header for frontend debugging
|
| 579 |
if (loadedImages && loadedImages.final) {
|
|
|
|
| 582 |
res.setHeader('X-Image-Count', String(loadedImages.final.length));
|
| 583 |
}
|
| 584 |
|
| 585 |
+
// 关闭 page + 释放浏览器回池(发送响应前完成,尽早释放 CPU/内存)
|
| 586 |
+
if (page) {
|
| 587 |
+
try { await page.close(); } catch (e) {}
|
| 588 |
+
page = null;
|
| 589 |
+
}
|
| 590 |
+
if (acquired) {
|
| 591 |
+
if (typeof acquired.release === 'function') {
|
| 592 |
+
try { await acquired.release(); } catch (e) {}
|
| 593 |
+
} else {
|
| 594 |
+
console.error(`[PDF-GEN] [${getElapsed()}] BUG: acquired slot has no release() method — pool will leak!`);
|
| 595 |
+
}
|
| 596 |
+
acquired = null;
|
| 597 |
+
}
|
| 598 |
browser = null;
|
| 599 |
|
| 600 |
console.log(`[PDF-GEN] [${getElapsed()}] >>> 任务全部完成 <<<`);
|
| 601 |
|
| 602 |
res.setHeader('Content-Type', 'application/pdf');
|
| 603 |
res.setHeader('Content-Disposition', 'attachment; filename=export.pdf');
|
| 604 |
+
// page.pdf() 返回 Uint8Array(非 Buffer)。Express 的 res.send() 只对真正的
|
| 605 |
+
// Buffer 走二进制路径,否则会 JSON 序列化({"0":37,"1":80,...})。
|
| 606 |
+
// 必须用 Buffer.from() 转换回 Buffer 才能正确返回二进制 PDF。
|
| 607 |
res.send(Buffer.from(pdfBuffer));
|
| 608 |
|
| 609 |
} catch (error) {
|
| 610 |
console.error(`[PDF-GEN] [${getElapsed()}] 发生错误:`, error);
|
| 611 |
+
// 释放池化浏览器(关闭 page 后归还/回收),避免资源泄漏
|
| 612 |
+
if (page) {
|
| 613 |
+
try { await page.close(); } catch (e) {}
|
| 614 |
+
page = null;
|
| 615 |
}
|
| 616 |
+
if (acquired) {
|
| 617 |
+
if (typeof acquired.release === 'function') {
|
| 618 |
+
try { await acquired.release(); } catch (e) {}
|
| 619 |
+
} else {
|
| 620 |
+
console.error(`[PDF-GEN] [${getElapsed()}] BUG: acquired slot has no release() method — pool will leak!`);
|
| 621 |
+
}
|
| 622 |
+
acquired = null;
|
| 623 |
+
}
|
| 624 |
+
browser = null;
|
| 625 |
// 清理临时文件
|
| 626 |
if (tempFilePath) {
|
| 627 |
+
try { await fs.promises.unlink(tempFilePath); } catch (e) {}
|
| 628 |
}
|
| 629 |
res.status(500).json({ error: 'Internal Server Error', details: error.message });
|
| 630 |
}
|
|
|
|
| 635 |
class WidgetRenderer {
|
| 636 |
constructor() {
|
| 637 |
this._chartInstances = new Map();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 638 |
}
|
| 639 |
|
| 640 |
+
/**
|
| 641 |
+
* 从 widget 浏览器池获取一个空闲浏览器(进程级隔离,等价稳定版每 widget
|
| 642 |
+
* 一个浏览器),返回 { browser, release }。调用方必须在 finally 中 release。
|
| 643 |
+
* 池满时排队,等待上限 WIDGET_ACQUIRE_TIMEOUT_MS。
|
| 644 |
+
*/
|
| 645 |
+
async acquireWidgetBrowser() {
|
| 646 |
+
return widgetPool.acquire();
|
| 647 |
}
|
| 648 |
|
| 649 |
_getChartInstance(width, height) {
|
|
|
|
| 955 |
* The HTML already contains full <html>/<head>/<body> with script tags.
|
| 956 |
*/
|
| 957 |
async _renderFullHtml(htmlContent, title, renderTimeout, chartType) {
|
| 958 |
+
// 与 renderWidgetPuppeteer 相同的硬超时包装,防止 CDP 挂起永久占用并发槽位
|
| 959 |
+
const HARD_TIMEOUT_MS = WIDGET_HARD_TIMEOUT_MS;
|
| 960 |
+
let timer = null;
|
| 961 |
+
try {
|
| 962 |
+
const timeoutPromise = new Promise((_, reject) => {
|
| 963 |
+
timer = setTimeout(
|
| 964 |
+
() => reject(new Error(`_renderFullHtml hard timeout after ${HARD_TIMEOUT_MS}ms`)),
|
| 965 |
+
HARD_TIMEOUT_MS
|
| 966 |
+
);
|
| 967 |
+
});
|
| 968 |
+
return await Promise.race([
|
| 969 |
+
this._renderFullHtmlImpl(htmlContent, title, renderTimeout, chartType),
|
| 970 |
+
timeoutPromise,
|
| 971 |
+
]);
|
| 972 |
+
} catch (e) {
|
| 973 |
+
console.log(`[WIDGET] _renderFullHtml ERROR: ${e.message}`);
|
| 974 |
+
return null;
|
| 975 |
+
} finally {
|
| 976 |
+
clearTimeout(timer);
|
| 977 |
+
}
|
| 978 |
+
}
|
| 979 |
+
|
| 980 |
+
async _renderFullHtmlImpl(htmlContent, title, renderTimeout, chartType) {
|
| 981 |
const startTime = Date.now();
|
| 982 |
console.log(`[WIDGET] _renderFullHtml START: title=${title}, timeout=${renderTimeout}ms, chartType=${chartType}`);
|
| 983 |
|
| 984 |
+
let page = null;
|
| 985 |
+
let acquired = null;
|
| 986 |
try {
|
| 987 |
+
// 从 widget 浏览器池获取浏览器(进程级隔离,避免单例浏览器争抢拖垮并发)
|
| 988 |
+
acquired = await this.acquireWidgetBrowser();
|
| 989 |
+
const browser = acquired.browser;
|
| 990 |
+
page = await browser.newPage();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 991 |
|
| 992 |
// CRITICAL: Use a compact viewport height to avoid excessive whitespace
|
| 993 |
// in fullPage screenshots. Width 800 gives enough room for 650px chart.
|
| 994 |
await page.setViewport({ width: 800, height: 500 });
|
| 995 |
|
| 996 |
// Set complete HTML document directly
|
| 997 |
+
// 性能优化:'load' 替代 networkidle0(避免 ~2s 额外等待)
|
| 998 |
+
await page.setContent(htmlContent, { waitUntil: 'load', timeout: 15000 });
|
| 999 |
+
await page.waitForNetworkIdle({ idleTime: 300, timeout: 15000 });
|
| 1000 |
|
| 1001 |
// Wait for React/Recharts to render (check window._widgetRendered flag)
|
| 1002 |
const rendered = await page.waitForFunction(
|
|
|
|
| 1027 |
console.log(`[WIDGET] _renderFullHtml ERROR: ${e.message}`);
|
| 1028 |
return null;
|
| 1029 |
} finally {
|
| 1030 |
+
// 只关闭 page,不关闭浏览器(浏览器归还给池复用)
|
| 1031 |
+
if (page) {
|
| 1032 |
+
try { await page.close(); } catch (e) {}
|
| 1033 |
+
}
|
| 1034 |
+
if (acquired && typeof acquired.release === 'function') {
|
| 1035 |
+
try { await acquired.release(); } catch (e) {}
|
| 1036 |
}
|
| 1037 |
}
|
| 1038 |
}
|
|
|
|
| 1447 |
}
|
| 1448 |
|
| 1449 |
async renderWidgetPuppeteer(widgetHtml, type, title, renderTimeout) {
|
| 1450 |
+
// 硬超时包装:整个渲染流程必须有界,否则 page 卡死时(页面主线程繁忙、
|
| 1451 |
+
// 渲染进程无响应)单个 widget 可能挂起数分钟,永久占用共享并发队列槽位,
|
| 1452 |
+
// 导致后续 widget 排队直至 30s 应用级超时、批量级联失败(DOCX 图表因此
|
| 1453 |
+
// fallback 成数据表格)。硬超时后立即返回 null(前端优雅降级),并在
|
| 1454 |
+
// _renderWidgetPuppeteerImpl 的 finally 中关闭 page 释放资源。
|
| 1455 |
+
const HARD_TIMEOUT_MS = WIDGET_HARD_TIMEOUT_MS;
|
| 1456 |
+
let timer = null;
|
| 1457 |
+
try {
|
| 1458 |
+
const timeoutPromise = new Promise((_, reject) => {
|
| 1459 |
+
timer = setTimeout(
|
| 1460 |
+
() => reject(new Error(`renderPuppeteer hard timeout after ${HARD_TIMEOUT_MS}ms`)),
|
| 1461 |
+
HARD_TIMEOUT_MS
|
| 1462 |
+
);
|
| 1463 |
+
});
|
| 1464 |
+
return await Promise.race([
|
| 1465 |
+
this._renderWidgetPuppeteerImpl(widgetHtml, type, title, renderTimeout),
|
| 1466 |
+
timeoutPromise,
|
| 1467 |
+
]);
|
| 1468 |
+
} catch (e) {
|
| 1469 |
+
console.log(`[WIDGET] renderPuppeteer ERROR: ${e.message}`);
|
| 1470 |
+
return null;
|
| 1471 |
+
} finally {
|
| 1472 |
+
clearTimeout(timer);
|
| 1473 |
+
}
|
| 1474 |
+
}
|
| 1475 |
+
|
| 1476 |
+
async _renderWidgetPuppeteerImpl(widgetHtml, type, title, renderTimeout) {
|
| 1477 |
const startTime = Date.now();
|
| 1478 |
console.log(`[WIDGET] renderPuppeteer START: type=${type}, title=${title}, timeout=${renderTimeout}ms`);
|
| 1479 |
|
| 1480 |
+
let page = null;
|
| 1481 |
+
let acquired = null;
|
| 1482 |
try {
|
| 1483 |
const finalTimeout = renderTimeout || 3000;
|
| 1484 |
|
| 1485 |
+
// 从 widget 浏览器池获取浏览器(进程级隔离,每个 widget 独占一个浏览器,
|
| 1486 |
+
// 避免单例浏览器下多个 widget 同时渲染争抢页面主线程导致挂起/级联超时)
|
| 1487 |
+
acquired = await this.acquireWidgetBrowser();
|
| 1488 |
+
const browser = acquired.browser;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1489 |
|
| 1490 |
+
page = await browser.newPage();
|
| 1491 |
await page.setViewport({ width: 1024, height: 768 });
|
| 1492 |
|
| 1493 |
const templatesDir = path.join(__dirname, 'templates');
|
|
|
|
| 1523 |
let fullHtml = template
|
| 1524 |
.replace('%%WIDGET_CODE%%', widgetCode);
|
| 1525 |
|
| 1526 |
+
// 性能优化:与 PDF 一致,setContent 用 'load'(避免 networkidle0 的 ~2s 额外等待)
|
| 1527 |
+
await page.setContent(fullHtml, { waitUntil: 'load', timeout: 15000 });
|
| 1528 |
+
await page.waitForNetworkIdle({ idleTime: 300, timeout: 15000 });
|
| 1529 |
|
| 1530 |
+
// 注意:不要在这里把 _widgetRendered 重置为 false。
|
| 1531 |
+
// 模板里的信号脚本在 setContent 解析期间(等 Chart.js CDN 阻塞加载完成后)
|
| 1532 |
+
// 就执行了 setTimeout(signalDone, 1500),即 _widgetRendered 通常在 load +
|
| 1533 |
+
// networkidle 之后已经是 true。此时再 evaluate 重置为 false,waitForFunction
|
| 1534 |
+
// 将永远等不到第二次置 true,只能干等满 finalTimeout(5s) 后超时,白白浪费
|
| 1535 |
+
// ~5s/widget,并显著拖慢高并发下的渲染(实测渲染耗时呈 2.5s / 5.6s 双峰)。
|
| 1536 |
+
// 模板本身已正确管理该信号,无需也不应重置。
|
| 1537 |
|
| 1538 |
let renderComplete = false;
|
| 1539 |
|
|
|
|
| 1669 |
console.log(`[WIDGET] renderPuppeteer ERROR: ${e.message}`);
|
| 1670 |
return null;
|
| 1671 |
} finally {
|
| 1672 |
+
// 只关闭 page,不关闭浏览器(浏览器归还给池复用)
|
| 1673 |
+
if (page) {
|
| 1674 |
+
try { await page.close(); } catch (e) {}
|
| 1675 |
+
}
|
| 1676 |
+
if (acquired && typeof acquired.release === 'function') {
|
| 1677 |
+
try { await acquired.release(); } catch (e) {}
|
| 1678 |
}
|
| 1679 |
}
|
| 1680 |
}
|
|
|
|
| 1702 |
}
|
| 1703 |
|
| 1704 |
const widgetRenderer = new WidgetRenderer();
|
| 1705 |
+
const MAX_CONCURRENT_RENDER = WIDGET_MAX_CONCURRENT;
|
| 1706 |
let activeRenderCount = 0;
|
| 1707 |
const renderQueue = [];
|
| 1708 |
|
| 1709 |
+
/**
|
| 1710 |
+
* 限制并发渲染,超出上限时在 renderQueue 中排队。
|
| 1711 |
+
* @param {object} widget
|
| 1712 |
+
* @param {number} timeoutMs 排队最长时间。超时后从队列移除并 reject,
|
| 1713 |
+
* 避免「应用级 30s 超时已返回但渲染仍在队列等待」产生僵尸条目 ——
|
| 1714 |
+
* 僵尸条目会一直占用队列位置,导致过载后的后续请求继续排队超时
|
| 1715 |
+
* (实测过载后首个恢复请求仍整批 30s 超时,之后才恢复)。
|
| 1716 |
+
*/
|
| 1717 |
+
function renderWithConcurrency(widget, timeoutMs) {
|
| 1718 |
return new Promise((resolve, reject) => {
|
| 1719 |
+
const entry = { widget, resolve, reject, timer: null };
|
| 1720 |
+
if (timeoutMs && timeoutMs > 0) {
|
| 1721 |
+
entry.timer = setTimeout(() => {
|
| 1722 |
+
const i = renderQueue.indexOf(entry);
|
| 1723 |
+
if (i !== -1) {
|
| 1724 |
+
renderQueue.splice(i, 1);
|
| 1725 |
+
reject(new Error('render queue wait timeout'));
|
| 1726 |
+
}
|
| 1727 |
+
// 已开始执行的 entry 不在队列中,由 renderWidgetPuppeteer 内部
|
| 1728 |
+
// 硬超时 + CDP 超时兜底,无需在此处理。
|
| 1729 |
+
}, timeoutMs);
|
| 1730 |
+
}
|
| 1731 |
+
renderQueue.push(entry);
|
| 1732 |
processRenderQueue();
|
| 1733 |
});
|
| 1734 |
}
|
| 1735 |
|
| 1736 |
async function processRenderQueue() {
|
| 1737 |
while (renderQueue.length > 0 && activeRenderCount < MAX_CONCURRENT_RENDER) {
|
| 1738 |
+
const entry = renderQueue.shift();
|
| 1739 |
+
if (entry.timer) clearTimeout(entry.timer);
|
| 1740 |
activeRenderCount++;
|
| 1741 |
try {
|
| 1742 |
+
const result = await widgetRenderer.render(entry.widget);
|
| 1743 |
+
entry.resolve(result);
|
| 1744 |
} catch (e) {
|
| 1745 |
+
entry.reject(e);
|
| 1746 |
} finally {
|
| 1747 |
activeRenderCount--;
|
| 1748 |
processRenderQueue();
|
|
|
|
| 1773 |
console.log(`[WIDGET] [${index + 1}/${widgets.length}] Rendering: type=${wType}, title=${wTitle}`);
|
| 1774 |
|
| 1775 |
try {
|
|
|
|
| 1776 |
// Per-widget timeout: mermaid needs extra time for CDN loading + rendering (~15s).
|
| 1777 |
// Other widgets complete in <5s. Using a generous 60s timeout for all types.
|
| 1778 |
const widgetTimeout = wType === 'mermaid' ? 60000 : 30000;
|
| 1779 |
+
// 排队等待超时同步传入队列,超时即从队列移除,避免僵尸条目阻塞后续请求
|
| 1780 |
+
const renderPromise = renderWithConcurrency(widget, widgetTimeout);
|
| 1781 |
const timeoutPromise = new Promise(resolve => setTimeout(() => resolve(null), widgetTimeout));
|
| 1782 |
const dataUrl = await Promise.race([renderPromise, timeoutPromise]);
|
| 1783 |
|
|
|
|
| 1806 |
res.json({ results: renderResults });
|
| 1807 |
});
|
| 1808 |
|
| 1809 |
+
// ─── Body-parser / JSON 解析错误兜底 ───────────────────────────
|
| 1810 |
+
// express.json() 解析失败(Content-Type: application/json 但 body 非法)时,
|
| 1811 |
+
// 会抛 entity.parse.failed 错误。外部探针/扫描器会定期向公开接口发送这类
|
| 1812 |
+
// 非法 JSON。若不兜底,Express 默认错误处理器会把完整堆栈打到 stdout
|
| 1813 |
+
// (刷屏 HF LOG)并在响应中暴露技术栈。这里返回干净的 400,不打印堆栈。
|
| 1814 |
+
app.use((err, req, res, next) => {
|
| 1815 |
+
if (err && err.type === 'entity.parse.failed') {
|
| 1816 |
+
return res.status(400).json({ error: 'Invalid JSON body' });
|
| 1817 |
+
}
|
| 1818 |
+
next(err);
|
| 1819 |
+
});
|
| 1820 |
+
|
| 1821 |
app.listen(port, () => {
|
| 1822 |
console.log(`Server listening at http://localhost:${port}`);
|
| 1823 |
});
|
tests/stress/README.md
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PDF 导出压力测试工具
|
| 2 |
+
|
| 3 |
+
用于测量 PDF 导出后端 (`/api/generate_pdf`) 的并发承载能力(同时能有多少用户导出 PDF)。
|
| 4 |
+
|
| 5 |
+
## 文件
|
| 6 |
+
|
| 7 |
+
| 文件 | 说明 |
|
| 8 |
+
|------|------|
|
| 9 |
+
| `payload-generator.js` | 生成确定性的仿真 PDF HTML 负载(文本 + Shiki 代码块 + base64 图片) |
|
| 10 |
+
| `run-stress-test.js` | 主压力测试脚本:按并发梯度打请求,统计成功率/延迟/吞吐 |
|
| 11 |
+
| `monitor-docker.js` | 并行监控 docker 容器 CPU/内存,与压力测试结果关联 |
|
| 12 |
+
| `verify-pdf-output.js` | 生成一份 PDF 保存到磁盘,验证输出为真实 PDF(`%PDF-1.x` 魔数) |
|
| 13 |
+
| `test-widget-batch.js` | 批量 widget 渲染验证(模拟 PDF/DOCX 导出的 `render_charts` 调用) |
|
| 14 |
+
| `results/` | 测试报告 JSON + 监控 JSON |
|
| 15 |
+
|
| 16 |
+
## 用法
|
| 17 |
+
|
| 18 |
+
```bash
|
| 19 |
+
# 基线/优化后对比测试(medium 负载)
|
| 20 |
+
node tests/stress/run-stress-test.js \
|
| 21 |
+
--base http://localhost:17861 \
|
| 22 |
+
--concurrency 1,2,4,6,8 \
|
| 23 |
+
--total 8 \
|
| 24 |
+
--profile medium \
|
| 25 |
+
--settle-ms 3000 \
|
| 26 |
+
--tag opt-pool4
|
| 27 |
+
|
| 28 |
+
# 并行监控 docker 资源(在另一个终端运行)
|
| 29 |
+
node tests/stress/monitor-docker.js \
|
| 30 |
+
--containers pdf-test,pdf-prod \
|
| 31 |
+
--interval 2000 \
|
| 32 |
+
--duration 240000 \
|
| 33 |
+
--out tests/stress/results/monitor-x.json
|
| 34 |
+
|
| 35 |
+
# 校验 PDF 输出质量
|
| 36 |
+
node tests/stress/verify-pdf-output.js
|
| 37 |
+
|
| 38 |
+
# 验证批量 widget 渲染
|
| 39 |
+
node tests/stress/test-widget-batch.js
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
## 参数
|
| 43 |
+
|
| 44 |
+
| 参数 | 默认值 | 说明 |
|
| 45 |
+
|------|--------|------|
|
| 46 |
+
| `--base` | `http://localhost:17861` | 后端地址(17861=测试容器,7860=生产容器) |
|
| 47 |
+
| `--concurrency` | `1,2,4,6,8` | 并发梯度(逗号分隔,每个梯度跑 `--total` 个请求) |
|
| 48 |
+
| `--total` | `20` | 每个并发梯度下总请求数 |
|
| 49 |
+
| `--profile` | `medium` | 负载规格:`small`(62KB) / `medium`(375KB+3图) / `large`(1.24MB+10图) |
|
| 50 |
+
| `--settle-ms` | `10000` | 每个梯度之间的冷却时间,让内存回落 |
|
| 51 |
+
| `--tag` | `` | 报告文件名后缀,用于区分基线/优化 |
|
| 52 |
+
| `--timeout-ms` | `180000` | 单请求超时 |
|
| 53 |
+
|
| 54 |
+
## 报告解读
|
| 55 |
+
|
| 56 |
+
- `throughputPerMin` 基于**墙钟时间**(该梯度内实际完成的请求数/分钟),反映系统真实吞吐。
|
| 57 |
+
- `latencyMs` 是每个请求从发起到收到响应的耗时(含排队等待)。
|
| 58 |
+
- 失败分类:`500`=服务端错误,`conn/...`=连接层错误。
|
| 59 |
+
|
| 60 |
+
## 重要提示
|
| 61 |
+
|
| 62 |
+
1. **本机 docker 不能完全模拟 Hugging Face**:HF 免费版 = 2 vCPU / 16GB;本机 docker 为 16 核。本机测出的并发上限是 HF 的**上界**,HF 上需按 `PDF_POOL_SIZE=2` 部署。
|
| 63 |
+
2. 测试前确认 `pdf-test` 容器已用最新代码重建。
|
| 64 |
+
3. 机器上还跑着其他系统(如 xianzhi-*),测试不要设置过大的并发,避免干扰。
|
tests/stress/analyze-hf-logs.js
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Analyze Hugging Face Space run logs for PDF/Widget export usage statistics.
|
| 3 |
+
*
|
| 4 |
+
* Parses the SSE stream returned by:
|
| 5 |
+
* curl -s -H "Authorization: Bearer $HF_TOKEN" \
|
| 6 |
+
* "https://huggingface.co/api/spaces/XWX-AI/api-server/logs/run"
|
| 7 |
+
*
|
| 8 |
+
* Extracts per-request telemetry (platform / version / language / export counts /
|
| 9 |
+
* message count / image count), per-request latency, and setContent timing.
|
| 10 |
+
* Writes a UTF-8 markdown report to tests/stress/results/.
|
| 11 |
+
*
|
| 12 |
+
* Usage:
|
| 13 |
+
* node tests/stress/analyze-hf-logs.js <raw-sse-file> [--out report.md]
|
| 14 |
+
*/
|
| 15 |
+
|
| 16 |
+
const fs = require('fs');
|
| 17 |
+
const path = require('path');
|
| 18 |
+
|
| 19 |
+
const parseArgs = (argv) => {
|
| 20 |
+
const args = { _: [] };
|
| 21 |
+
for (let i = 0; i < argv.length; i++) {
|
| 22 |
+
const m = argv[i].match(/^--([^=]+)(?:=(.*))?$/);
|
| 23 |
+
if (!m) { args._.push(argv[i]); continue; }
|
| 24 |
+
if (m[2] !== undefined) {
|
| 25 |
+
args[m[1]] = m[2];
|
| 26 |
+
} else if (i + 1 < argv.length && !argv[i + 1].startsWith('--')) {
|
| 27 |
+
args[m[1]] = argv[++i];
|
| 28 |
+
} else {
|
| 29 |
+
args[m[1]] = true;
|
| 30 |
+
}
|
| 31 |
+
}
|
| 32 |
+
return args;
|
| 33 |
+
};
|
| 34 |
+
|
| 35 |
+
// ─── log line helpers (all server log lines are Chinese; match by unique ASCII anchors) ───
|
| 36 |
+
|
| 37 |
+
const RE_TELEMETRY = /收到请求 \| 平台: (\S+) \| 版本: (\S+) \| 语言: (\S+)/;
|
| 38 |
+
const RE_EXPORT_SUMMARY = /导出: (\d+)次 \| 格式: PDF:(\d+), MD:(\d+), TXT:(\d+), DOCX:(\d+), JSON:(\d+), CLIP:(\d+), NOTION:(\d+) \| 消息: (\d+)条 \| 图片: (\d+)张/;
|
| 39 |
+
const RE_DONE = />>> 任务全部完成 <<</;
|
| 40 |
+
const RE_CONTENT_LOADED = /页面内容加载完成/;
|
| 41 |
+
const RE_PDF_START = /正在生成 PDF 二进制流/;
|
| 42 |
+
const RE_HTML_PARSE = /解析请求完成: HTML ([\d.]+) MB/;
|
| 43 |
+
const RE_LAUNCH_READY = /浏览器启动成功/;
|
| 44 |
+
const RE_FILLING = /正在填充页面内容|正在通过临时文件加载页面内容/;
|
| 45 |
+
const RE_RENDER_DONE = /render_charts DONE: (\d+) OK, (\d+) FAIL, ([\d.]+)s/;
|
| 46 |
+
const RE_BROWSER_LAUNCH = /正在从浏览器池获取浏览器|正在启动浏览器/;
|
| 47 |
+
|
| 48 |
+
function hourOf(ts) {
|
| 49 |
+
const d = new Date(ts);
|
| 50 |
+
return d.getUTCHours();
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
function main() {
|
| 54 |
+
const args = parseArgs(process.argv.slice(2));
|
| 55 |
+
const inFile = args._ ? args._[0] : null;
|
| 56 |
+
if (!inFile) {
|
| 57 |
+
console.error('Usage: node tests/stress/analyze-hf-logs.js <sse-log-file> [--out report.md]');
|
| 58 |
+
process.exit(1);
|
| 59 |
+
}
|
| 60 |
+
const raw = fs.readFileSync(inFile, 'utf8');
|
| 61 |
+
|
| 62 |
+
const requests = []; // one entry per completed PDF-GEN request
|
| 63 |
+
let pending = null;
|
| 64 |
+
let renderCalls = [];
|
| 65 |
+
|
| 66 |
+
for (const line of raw.split('\n')) {
|
| 67 |
+
if (!line.startsWith('data: ')) continue;
|
| 68 |
+
let obj;
|
| 69 |
+
try { obj = JSON.parse(line.slice(6)); } catch (e) { continue; }
|
| 70 |
+
const msg = obj.data;
|
| 71 |
+
if (!msg) continue;
|
| 72 |
+
|
| 73 |
+
const t = obj.timestamp;
|
| 74 |
+
|
| 75 |
+
const tele = msg.match(RE_TELEMETRY);
|
| 76 |
+
if (tele) {
|
| 77 |
+
pending = {
|
| 78 |
+
ts: t,
|
| 79 |
+
platform: tele[1],
|
| 80 |
+
version: tele[2],
|
| 81 |
+
language: tele[3],
|
| 82 |
+
exports: null, htmlMB: null, latencyMs: null, setContentMs: null, pdfMs: null, launchMs: null,
|
| 83 |
+
};
|
| 84 |
+
continue;
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
const sum = msg.match(RE_EXPORT_SUMMARY);
|
| 88 |
+
if (sum && pending) {
|
| 89 |
+
pending.exports = {
|
| 90 |
+
total: +sum[1], pdf: +sum[2], md: +sum[3], txt: +sum[4], docx: +sum[5], json: +sum[6], clip: +sum[7], notion: +sum[8],
|
| 91 |
+
messages: +sum[9], images: +sum[10],
|
| 92 |
+
};
|
| 93 |
+
continue;
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
const hp = msg.match(RE_HTML_PARSE);
|
| 97 |
+
if (hp && pending) {
|
| 98 |
+
pending.htmlMB = parseFloat(hp[1]);
|
| 99 |
+
continue;
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
if (RE_BROWSER_LAUNCH.test(msg) && pending && pending.launchMs === null) {
|
| 103 |
+
pending._launchT = new Date(t).getTime();
|
| 104 |
+
continue;
|
| 105 |
+
}
|
| 106 |
+
if (RE_LAUNCH_READY.test(msg) && pending && pending._launchT) {
|
| 107 |
+
pending.launchMs = new Date(t).getTime() - pending._launchT;
|
| 108 |
+
continue;
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
if (RE_FILLING.test(msg) && pending && pending._fillT === undefined) {
|
| 112 |
+
pending._fillT = new Date(t).getTime();
|
| 113 |
+
continue;
|
| 114 |
+
}
|
| 115 |
+
if (RE_CONTENT_LOADED.test(msg) && pending && pending._fillT !== undefined && pending.setContentMs === null) {
|
| 116 |
+
// setContent 阶段 = 开始填充页面内容 → 页面内容加载完成(含 networkidle 等待)
|
| 117 |
+
pending.setContentMs = new Date(t).getTime() - pending._fillT;
|
| 118 |
+
continue;
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
if (RE_PDF_START.test(msg) && pending && pending.pdfMs === null) {
|
| 122 |
+
pending._pdfStart = new Date(t).getTime();
|
| 123 |
+
continue;
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
if (RE_DONE.test(msg) && pending) {
|
| 127 |
+
pending.latencyMs = new Date(t).getTime() - new Date(pending.ts).getTime();
|
| 128 |
+
if (pending._pdfStart) pending.pdfMs = new Date(t).getTime() - pending._pdfStart;
|
| 129 |
+
requests.push(pending);
|
| 130 |
+
pending = null;
|
| 131 |
+
continue;
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
const rd = msg.match(RE_RENDER_DONE);
|
| 135 |
+
if (rd) {
|
| 136 |
+
renderCalls.push({ ts: t, ok: +rd[1], fail: +rd[2], sec: parseFloat(rd[3]) });
|
| 137 |
+
}
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
// ─── aggregate ───
|
| 141 |
+
const total = requests.length;
|
| 142 |
+
const byPlatform = {};
|
| 143 |
+
const byVersion = {};
|
| 144 |
+
const byLanguage = {};
|
| 145 |
+
const byHour = {};
|
| 146 |
+
const pdfExports = { total: 0 };
|
| 147 |
+
for (const r of requests) {
|
| 148 |
+
byPlatform[r.platform] = (byPlatform[r.platform] || 0) + 1;
|
| 149 |
+
byVersion[r.version] = (byVersion[r.version] || 0) + 1;
|
| 150 |
+
byLanguage[r.language] = (byLanguage[r.language] || 0) + 1;
|
| 151 |
+
const h = hourOf(r.ts);
|
| 152 |
+
byHour[h] = (byHour[h] || 0) + 1;
|
| 153 |
+
if (r.exports) {
|
| 154 |
+
for (const k of ['pdf', 'md', 'txt', 'docx', 'json', 'clip', 'notion']) {
|
| 155 |
+
pdfExports[k] = (pdfExports[k] || 0) + r.exports[k];
|
| 156 |
+
}
|
| 157 |
+
pdfExports.total += r.exports.total;
|
| 158 |
+
}
|
| 159 |
+
}
|
| 160 |
+
|
| 161 |
+
const lat = requests.map((r) => r.latencyMs).filter((v) => v != null).sort((a, b) => a - b);
|
| 162 |
+
const sc = requests.map((r) => r.setContentMs).filter((v) => v != null).sort((a, b) => a - b);
|
| 163 |
+
const pm = requests.map((r) => r.pdfMs).filter((v) => v != null).sort((a, b) => a - b);
|
| 164 |
+
const lm = requests.map((r) => r.launchMs).filter((v) => v != null).sort((a, b) => a - b);
|
| 165 |
+
const pct = (arr, p) => arr.length ? arr[Math.min(arr.length - 1, Math.ceil((p / 100) * arr.length) - 1)] : 0;
|
| 166 |
+
const avg = (arr) => arr.length ? Math.round(arr.reduce((a, b) => a + b, 0) / arr.length) : 0;
|
| 167 |
+
|
| 168 |
+
const ts = requests.map((r) => new Date(r.ts).getTime()).sort((a, b) => a - b);
|
| 169 |
+
const spanH = ts.length >= 2 ? ((ts[ts.length - 1] - ts[0]) / 3600000) : 0;
|
| 170 |
+
|
| 171 |
+
// ─── report ───
|
| 172 |
+
const lines = [];
|
| 173 |
+
lines.push('# Hugging Face 生产日志分析');
|
| 174 |
+
lines.push('');
|
| 175 |
+
lines.push(`分析时间: ${new Date().toISOString()}`);
|
| 176 |
+
lines.push(`日志文件: ${path.basename(inFile)}`);
|
| 177 |
+
lines.push('');
|
| 178 |
+
lines.push('## 总览');
|
| 179 |
+
lines.push('');
|
| 180 |
+
lines.push(`| 指标 | 值 |`);
|
| 181 |
+
lines.push(`|------|-----|`);
|
| 182 |
+
lines.push(`| 完整 PDF 请求数 | ${total} |`);
|
| 183 |
+
lines.push(`| 时间跨度 | ${spanH.toFixed(1)} 小时 |`);
|
| 184 |
+
lines.push(`| 平均每请求延迟 | ${avg(lat)} ms |`);
|
| 185 |
+
lines.push(`| 延迟 p50 / p95 | ${pct(lat, 50)} / ${pct(lat, 95)} ms |`);
|
| 186 |
+
lines.push(`| setContent 平均 | ${avg(sc)} ms(${sc.length} 个样本) |`);
|
| 187 |
+
lines.push(`| page.pdf() 平均 | ${avg(pm)} ms(${pm.length} 个样本) |`);
|
| 188 |
+
lines.push(`| 浏览器启动平均 | ${avg(lm)} ms(${lm.length} 个样本) |`);
|
| 189 |
+
lines.push(`| 累计导出格式 | ${Object.entries(pdfExports).map(([k, v]) => `${k}:${v}`).join(',')} |`);
|
| 190 |
+
lines.push(`| render_charts 调用 | ${renderCalls.length} 次,成功 ${renderCalls.reduce((a, c) => a + c.ok, 0)},失败 ${renderCalls.reduce((a, c) => a + c.fail, 0)} |`);
|
| 191 |
+
lines.push('');
|
| 192 |
+
lines.push('## 平台分布');
|
| 193 |
+
lines.push('');
|
| 194 |
+
lines.push('| 平台 | 请求数 |');
|
| 195 |
+
lines.push('|------|--------|');
|
| 196 |
+
for (const [k, v] of Object.entries(byPlatform).sort((a, b) => b[1] - a[1])) lines.push(`| ${k} | ${v} |`);
|
| 197 |
+
lines.push('');
|
| 198 |
+
lines.push('## 插件版本分布');
|
| 199 |
+
lines.push('');
|
| 200 |
+
lines.push('| 版本 | 请求数 |');
|
| 201 |
+
lines.push('|------|--------|');
|
| 202 |
+
for (const [k, v] of Object.entries(byVersion).sort((a, b) => b[1] - a[1])) lines.push(`| ${k} | ${v} |`);
|
| 203 |
+
lines.push('');
|
| 204 |
+
lines.push('## 语言分布');
|
| 205 |
+
lines.push('');
|
| 206 |
+
lines.push('| 语言 | 请求数 |');
|
| 207 |
+
lines.push('|------|--------|');
|
| 208 |
+
for (const [k, v] of Object.entries(byLanguage).sort((a, b) => b[1] - a[1])) lines.push(`| ${k} | ${v} |`);
|
| 209 |
+
lines.push('');
|
| 210 |
+
lines.push('## 小时分布 (UTC)');
|
| 211 |
+
lines.push('');
|
| 212 |
+
lines.push('| 小时 | 请求数 |');
|
| 213 |
+
lines.push('|------|--------|');
|
| 214 |
+
for (let h = 0; h < 24; h++) if (byHour[h]) lines.push(`| ${h} | ${byHour[h]} |`);
|
| 215 |
+
lines.push('');
|
| 216 |
+
lines.push('## 明细(最近 40 条)');
|
| 217 |
+
lines.push('');
|
| 218 |
+
lines.push('| 时间 (UTC) | 平台 | 版本 | 语言 | 延迟 | setContent | PDF | HTML MB | 格式 |');
|
| 219 |
+
lines.push('|-----------|------|------|------|------|-----------|-----|---------|------|');
|
| 220 |
+
const sorted = [...requests].sort((a, b) => b.ts.localeCompare(a.ts)).slice(0, 40);
|
| 221 |
+
for (const r of sorted) {
|
| 222 |
+
const e = r.exports;
|
| 223 |
+
lines.push(`| ${r.ts} | ${r.platform} | ${r.version} | ${r.language} | ${r.latencyMs ?? '-'}ms | ${r.setContentMs ?? '-'}ms | ${r.pdfMs ?? '-'}ms | ${r.htmlMB ?? '-'} | ${e ? `PDF:${e.pdf} MD:${e.md} DOCX:${e.docx} NOTION:${e.notion}` : '-'} |`);
|
| 224 |
+
}
|
| 225 |
+
|
| 226 |
+
const out = args.out || path.join(__dirname, 'results', `hf-analysis-${new Date().toISOString().slice(0, 10)}.md`);
|
| 227 |
+
if (!fs.existsSync(path.dirname(out))) fs.mkdirSync(path.dirname(out), { recursive: true });
|
| 228 |
+
fs.writeFileSync(out, lines.join('\n'), 'utf8');
|
| 229 |
+
console.log(`Report written: ${out}`);
|
| 230 |
+
console.log(`Requests: ${total} | avg latency: ${avg(lat)}ms | p95: ${pct(lat, 95)}ms | avg setContent: ${avg(sc)}ms | avg pdf(): ${avg(pm)}ms | exports: ${JSON.stringify(pdfExports)}`);
|
| 231 |
+
console.log(`Platforms: ${JSON.stringify(byPlatform)}`);
|
| 232 |
+
console.log(`Versions: ${JSON.stringify(byVersion)}`);
|
| 233 |
+
console.log(`Languages: ${JSON.stringify(byLanguage)}`);
|
| 234 |
+
console.log(`Hours(UTC): ${JSON.stringify(byHour)}`);
|
| 235 |
+
console.log(`render_charts: ${renderCalls.length} calls`);
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
+
main();
|
tests/stress/monitor-docker.js
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Monitors docker container CPU/memory during a stress test.
|
| 3 |
+
*
|
| 4 |
+
* Usage (run in parallel with run-stress-test.js):
|
| 5 |
+
* node stress/monitor-docker.js --containers pdf-test --interval 1000 --duration 300000 --out stress/results/monitor.json
|
| 6 |
+
*/
|
| 7 |
+
|
| 8 |
+
const { spawnSync } = require('child_process');
|
| 9 |
+
const fs = require('fs');
|
| 10 |
+
const path = require('path');
|
| 11 |
+
|
| 12 |
+
function parseArgs(argv) {
|
| 13 |
+
const args = { containers: 'pdf-test,pdf-prod', interval: 1000, duration: 300000, out: '' };
|
| 14 |
+
for (let i = 0; i < argv.length; i++) {
|
| 15 |
+
const m = argv[i].match(/^--([^=]+)(?:=(.*))?$/);
|
| 16 |
+
if (!m) continue;
|
| 17 |
+
if (m[2] !== undefined) {
|
| 18 |
+
args[m[1]] = m[2];
|
| 19 |
+
} else if (i + 1 < argv.length && !argv[i + 1].startsWith('--')) {
|
| 20 |
+
args[m[1]] = argv[++i];
|
| 21 |
+
} else {
|
| 22 |
+
args[m[1]] = true;
|
| 23 |
+
}
|
| 24 |
+
}
|
| 25 |
+
return args;
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
function sample(containers) {
|
| 29 |
+
try {
|
| 30 |
+
// Use spawnSync with an args array (no shell) to avoid Windows quoting bugs.
|
| 31 |
+
const res = spawnSync('docker', ['stats', '--no-stream', '--format', '{{json .}}', '--', ...containers], { encoding: 'utf8', timeout: 10000 });
|
| 32 |
+
if (res.status !== 0 || !res.stdout) return {};
|
| 33 |
+
const rows = {};
|
| 34 |
+
for (const line of res.stdout.split('\n').filter(Boolean)) {
|
| 35 |
+
let obj;
|
| 36 |
+
try { obj = JSON.parse(line); } catch (e) { continue; }
|
| 37 |
+
const name = obj.Name;
|
| 38 |
+
if (!name) continue;
|
| 39 |
+
rows[name] = {
|
| 40 |
+
cpuPerc: parseFloat(obj.CPUPerc) || 0,
|
| 41 |
+
memUsedGiB: parseGiB(obj.MemUsage),
|
| 42 |
+
memTotalGiB: 0,
|
| 43 |
+
memPerc: parseFloat(obj.MemPerc) || 0,
|
| 44 |
+
};
|
| 45 |
+
}
|
| 46 |
+
return rows;
|
| 47 |
+
} catch (e) {
|
| 48 |
+
return {};
|
| 49 |
+
}
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
function parseGiB(memUsageStr) {
|
| 53 |
+
if (!memUsageStr) return 0;
|
| 54 |
+
const m = memUsageStr.match(/^([\d.]+)\s*(\w+)\s*\/\s*([\d.]+)\s*(\w+)$/);
|
| 55 |
+
if (!m) return 0;
|
| 56 |
+
const used = parseFloat(m[1]);
|
| 57 |
+
const unit = m[2];
|
| 58 |
+
if (unit === 'GiB') return used;
|
| 59 |
+
if (unit === 'MiB') return used / 1024;
|
| 60 |
+
if (unit === 'KiB') return used / (1024 * 1024);
|
| 61 |
+
return used;
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
async function main() {
|
| 65 |
+
const args = parseArgs(process.argv.slice(2));
|
| 66 |
+
const containers = args.containers.split(',');
|
| 67 |
+
const interval = parseInt(args.interval, 10);
|
| 68 |
+
const duration = parseInt(args.duration, 10);
|
| 69 |
+
const outFile = args.out || path.join(__dirname, 'results', `monitor-${Date.now()}.json`);
|
| 70 |
+
|
| 71 |
+
const samples = [];
|
| 72 |
+
const started = Date.now();
|
| 73 |
+
console.log(`Monitoring ${containers.join(',')} for ${duration}ms...`);
|
| 74 |
+
while (Date.now() - started < duration) {
|
| 75 |
+
const rows = sample(containers);
|
| 76 |
+
samples.push({ t: Date.now() - started, rows });
|
| 77 |
+
process.stdout.write(`\rt=${Date.now() - started}ms ` + containers.map((c) => {
|
| 78 |
+
const r = rows[c];
|
| 79 |
+
return r ? `${c}: cpu=${r.cpuPerc.toFixed(1)}% mem=${r.memUsedGiB.toFixed(2)}GiB` : `${c}: n/a`;
|
| 80 |
+
}).join(' '));
|
| 81 |
+
await new Promise((r) => setTimeout(r, interval));
|
| 82 |
+
}
|
| 83 |
+
process.stdout.write('\n');
|
| 84 |
+
|
| 85 |
+
const summary = {};
|
| 86 |
+
for (const c of containers) {
|
| 87 |
+
const vals = samples.map((s) => s.rows[c]).filter(Boolean);
|
| 88 |
+
if (!vals.length) { summary[c] = null; continue; }
|
| 89 |
+
const cpu = vals.map((v) => v.cpuPerc).sort((a, b) => a - b);
|
| 90 |
+
const mem = vals.map((v) => v.memUsedGiB).sort((a, b) => a - b);
|
| 91 |
+
summary[c] = {
|
| 92 |
+
cpuPerc: { avg: +(cpu.reduce((a, b) => a + b, 0) / cpu.length).toFixed(1), peak: Math.round(cpu[cpu.length - 1]) },
|
| 93 |
+
memUsedGiB: { avg: +(mem.reduce((a, b) => a + b, 0) / mem.length).toFixed(2), peak: +mem[mem.length - 1].toFixed(2) },
|
| 94 |
+
};
|
| 95 |
+
}
|
| 96 |
+
const report = { started, duration, interval, containers, summary, samples };
|
| 97 |
+
fs.writeFileSync(outFile, JSON.stringify(report, null, 2));
|
| 98 |
+
console.log(`\nMonitor report saved: ${outFile}`);
|
| 99 |
+
console.log(JSON.stringify(summary, null, 2));
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
main().catch((e) => {
|
| 103 |
+
console.error(e);
|
| 104 |
+
process.exit(1);
|
| 105 |
+
});
|
tests/stress/payload-generator.js
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Stress test payload generator for PDF export benchmarking.
|
| 3 |
+
*
|
| 4 |
+
* Generates a realistic XWX AI Chat Exporter PDF HTML document:
|
| 5 |
+
* - chat message bubbles (AI + user)
|
| 6 |
+
* - markdown-rendered content: headings, paragraphs, tables
|
| 7 |
+
* - Shiki-style syntax highlighted code blocks (default 'github' theme)
|
| 8 |
+
* - optional base64 images (small PNGs, deterministic)
|
| 9 |
+
*
|
| 10 |
+
* Deterministic output: same profile => byte-identical HTML, so before/after
|
| 11 |
+
* optimization runs are comparable.
|
| 12 |
+
*/
|
| 13 |
+
|
| 14 |
+
const crypto = require('crypto');
|
| 15 |
+
const fs = require('fs');
|
| 16 |
+
const path = require('path');
|
| 17 |
+
|
| 18 |
+
const CSS = `
|
| 19 |
+
@media print { @page { size: A4; margin: 15mm 10mm; } body { -webkit-print-color-adjust: exact; } }
|
| 20 |
+
body { font-family: -apple-system, sans-serif; font-size: 14px; line-height: 1.6; max-width: 746px; margin: 0 auto; padding: 20px; }
|
| 21 |
+
h1,h2,h3 { font-weight: 600; margin: 16px 0 8px; }
|
| 22 |
+
pre { background: #f6f8fa; padding: 16px; border-radius: 6px; overflow-x: auto; border: 1px solid #e1e4e8; }
|
| 23 |
+
code { font-family: monospace; font-size: 13px; }
|
| 24 |
+
p { margin: 8px 0; }
|
| 25 |
+
table { border-collapse: collapse; width: 100%; margin: 12px 0; }
|
| 26 |
+
th,td { border: 1px solid #dfe2e5; padding: 8px 12px; } th { background: #f1f3f4; }
|
| 27 |
+
.chat-container { display: flex; flex-direction: column; gap: 16px; }
|
| 28 |
+
.message-row { display: flex; gap: 10px; }
|
| 29 |
+
.message-bubble { max-width: 85%; padding: 12px 16px; border-radius: 12px; }
|
| 30 |
+
.ai-bubble { background: #fff; border: 1px solid #eee; }
|
| 31 |
+
.user-bubble { background: #e8f0fe; }
|
| 32 |
+
.avatar { width: 32px; height: 32px; border-radius: 50%; display: flex; align-items: center; justify-content: center; }
|
| 33 |
+
`.trim();
|
| 34 |
+
|
| 35 |
+
function escapeHtml(s) {
|
| 36 |
+
return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
const CODE_LINES = [
|
| 40 |
+
'async function fetchData(url, options = {}) {',
|
| 41 |
+
' const controller = new AbortController();',
|
| 42 |
+
' const timeout = setTimeout(() => controller.abort(), 30000);',
|
| 43 |
+
' try {',
|
| 44 |
+
' const response = await fetch(url, {',
|
| 45 |
+
' ...options,',
|
| 46 |
+
' signal: controller.signal,',
|
| 47 |
+
" headers: { 'Content-Type': 'application/json' },",
|
| 48 |
+
' });',
|
| 49 |
+
' if (!response.ok) {',
|
| 50 |
+
' throw new Error(`HTTP ${response.status}: ${response.statusText}`);',
|
| 51 |
+
' }',
|
| 52 |
+
' const data = await response.json();',
|
| 53 |
+
" console.log('Data received:', data);",
|
| 54 |
+
' return data;',
|
| 55 |
+
' } catch (error) {',
|
| 56 |
+
" console.error('Fetch failed:', error.message);",
|
| 57 |
+
' throw error;',
|
| 58 |
+
' } finally {',
|
| 59 |
+
' clearTimeout(timeout);',
|
| 60 |
+
' }',
|
| 61 |
+
'}',
|
| 62 |
+
];
|
| 63 |
+
|
| 64 |
+
function shikiWrap(token, color) {
|
| 65 |
+
return `<span style="color:${color}">${escapeHtml(token)}</span>`;
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
// Deterministic pseudo-random generator (mulberry32) so payloads are stable.
|
| 69 |
+
function mulberry32(seed) {
|
| 70 |
+
return function () {
|
| 71 |
+
let t = (seed += 0x6d2b79f5);
|
| 72 |
+
t = Math.imul(t ^ (t >>> 15), t | 1);
|
| 73 |
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
| 74 |
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
| 75 |
+
};
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
function generateShikiCode(rng) {
|
| 79 |
+
return CODE_LINES.map((line) => {
|
| 80 |
+
const tokens = line.split(/(\s+|[^a-zA-Z0-9_\s]+)/g);
|
| 81 |
+
return tokens.map((tok) => {
|
| 82 |
+
if (!tok) return '';
|
| 83 |
+
if (/^\s+$/.test(tok)) return tok;
|
| 84 |
+
let color = '#24292e';
|
| 85 |
+
if (/^(const|let|var|function|return|if|else|for|async|await|import|from|export|class|extends|new|try|catch|throw|typeof|this|switch|case|break|continue|while|do|of|in|static|get|set|super)$/.test(tok)) color = '#d73a49';
|
| 86 |
+
else if (/^(console|log|error|fetch|Promise|Math|Date|JSON|Map|Set|Array|Object|String|Number|Error|setTimeout|clearTimeout|AbortController|AbortSignal|require|module|exports|process)$/.test(tok)) color = '#6f42c1';
|
| 87 |
+
else if (/^\d+$/.test(tok)) color = '#005cc5';
|
| 88 |
+
else if (/^[{}()\[\];,.:=+\-*/<>!&|?%@~^'"`]+$/.test(tok)) color = '#24292e';
|
| 89 |
+
else if (/^[A-Z]/.test(tok) && tok.length > 1) color = '#e36209';
|
| 90 |
+
return shikiWrap(tok, color);
|
| 91 |
+
}).join('');
|
| 92 |
+
}).join('\n');
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
// Tiny 1x1 PNG base64 (repeated to simulate distinct images)
|
| 96 |
+
const TINY_PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
|
| 97 |
+
|
| 98 |
+
function buildParagraph(rng, words) {
|
| 99 |
+
const vocab = [
|
| 100 |
+
'性能', '优化', '浏览器', '渲染', 'PDF', '导出', '并发', '线程', '内存',
|
| 101 |
+
'缓存', '请求', '服务器', 'docker', 'huggingface', 'puppeteer', 'chromium',
|
| 102 |
+
'the', 'quick', 'brown', 'fox', 'jumps', 'over', 'lazy', 'dog',
|
| 103 |
+
'data', 'stream', 'buffer', 'async', 'await', 'promise', 'queue', 'worker',
|
| 104 |
+
];
|
| 105 |
+
const parts = [];
|
| 106 |
+
for (let i = 0; i < words; i++) {
|
| 107 |
+
parts.push(vocab[Math.floor(rng() * vocab.length)]);
|
| 108 |
+
}
|
| 109 |
+
return `<p>${parts.join(' ')}</p>`;
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
/**
|
| 113 |
+
* @param {object} opts
|
| 114 |
+
* @param {number} opts.textMB target text-only size in MB
|
| 115 |
+
* @param {number} opts.images number of base64 images to embed
|
| 116 |
+
* @param {number} opts.codeRatio fraction (0-1) of content that is code blocks
|
| 117 |
+
* @param {number} opts.messages number of chat messages
|
| 118 |
+
*/
|
| 119 |
+
function buildHtml(opts = {}) {
|
| 120 |
+
const textMB = opts.textMB ?? 0.3;
|
| 121 |
+
const images = opts.images ?? 0;
|
| 122 |
+
const codeRatio = opts.codeRatio ?? 0.35;
|
| 123 |
+
const rng = mulberry32(0x5eed1234);
|
| 124 |
+
|
| 125 |
+
const codeBlock = `<pre data-language="javascript"><code class="language-javascript">${generateShikiCode(rng)}</code></pre>`;
|
| 126 |
+
const codeBlockSize = Buffer.byteLength(codeBlock, 'utf8');
|
| 127 |
+
const paraWords = 40;
|
| 128 |
+
const paraBlock = buildParagraph(rng, paraWords);
|
| 129 |
+
const paraBlockSize = Buffer.byteLength(paraBlock, 'utf8');
|
| 130 |
+
|
| 131 |
+
const overhead = 4096;
|
| 132 |
+
const targetBytes = Math.max(2048, textMB * 1024 * 1024 - overhead);
|
| 133 |
+
const codeBytes = targetBytes * codeRatio;
|
| 134 |
+
const textBytes = targetBytes * (1 - codeRatio);
|
| 135 |
+
const codeIters = Math.max(1, Math.ceil(codeBytes / codeBlockSize));
|
| 136 |
+
const textIters = Math.max(1, Math.ceil(textBytes / paraBlockSize));
|
| 137 |
+
|
| 138 |
+
const imageTag = `<img src="data:image/png;base64,${TINY_PNG_B64}" alt="i">`;
|
| 139 |
+
|
| 140 |
+
let html = `<!DOCTYPE html><html><head><meta charset="UTF-8"><style>${CSS}</style></head><body><div class="chat-container">`;
|
| 141 |
+
|
| 142 |
+
let i = 0, j = 0;
|
| 143 |
+
while (i < codeIters || j < textIters) {
|
| 144 |
+
if (i < codeIters) {
|
| 145 |
+
html += `<div class="message-row"><div class="avatar">🤖</div><div class="message-bubble ai-bubble">${codeBlock}</div></div>`;
|
| 146 |
+
i++;
|
| 147 |
+
}
|
| 148 |
+
if (j < textIters) {
|
| 149 |
+
html += `<div class="message-row"><div class="avatar">👤</div><div class="message-bubble user-bubble">${paraBlock}</div></div>`;
|
| 150 |
+
j++;
|
| 151 |
+
}
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
// Append images at the end (outside the loop so they don't blow up text size).
|
| 155 |
+
for (let k = 0; k < images; k++) {
|
| 156 |
+
html += `<div class="message-row"><div class="avatar">🤖</div><div class="message-bubble ai-bubble">${imageTag}</div></div>`;
|
| 157 |
+
}
|
| 158 |
+
|
| 159 |
+
html += '</div></body></html>';
|
| 160 |
+
return html;
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
/**
|
| 164 |
+
* Build a full request body for /api/generate_pdf.
|
| 165 |
+
* @param {object} opts same opts as buildHtml
|
| 166 |
+
* @param {object} meta optional metadata overrides
|
| 167 |
+
*/
|
| 168 |
+
function buildPdfRequest(opts = {}, meta = {}) {
|
| 169 |
+
const html = buildHtml(opts);
|
| 170 |
+
const textOnlySizeMB = Buffer.byteLength(html, 'utf8') / (1024 * 1024);
|
| 171 |
+
const images = opts.images ?? 0;
|
| 172 |
+
return {
|
| 173 |
+
html,
|
| 174 |
+
textOnlySizeMB,
|
| 175 |
+
codeTheme: meta.codeTheme ?? 'github',
|
| 176 |
+
showWatermark: meta.showWatermark ?? false,
|
| 177 |
+
imageCount: images,
|
| 178 |
+
totalImageSizeMB: (images * 1) / (1024 * 1024), // 1x1 png ~ 70 bytes
|
| 179 |
+
messageCount: meta.messageCount ?? 50,
|
| 180 |
+
platform: meta.platform ?? 'StressTest',
|
| 181 |
+
language: 'en-US',
|
| 182 |
+
extensionVersion: '2.1.6',
|
| 183 |
+
exportCount: 0, exportPdf: 0, exportMd: 0,
|
| 184 |
+
exportTxt: 0, exportDocx: 0, exportJson: 0,
|
| 185 |
+
exportClipboard: 0, exportNotion: 0,
|
| 186 |
+
};
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
/**
|
| 190 |
+
* Deterministically generate and cache a payload profile to a file.
|
| 191 |
+
* Returns { html, requestBody, sizeMB }.
|
| 192 |
+
*/
|
| 193 |
+
function prepareProfile(profile, outDir = path.join(__dirname, 'results')) {
|
| 194 |
+
const hash = crypto.createHash('md5').update(JSON.stringify(profile)).digest('hex').slice(0, 10);
|
| 195 |
+
const file = path.join(outDir, `payload-${hash}.json`);
|
| 196 |
+
if (fs.existsSync(file)) {
|
| 197 |
+
const cached = JSON.parse(fs.readFileSync(file, 'utf8'));
|
| 198 |
+
cached.requestBody.html = cached.html;
|
| 199 |
+
return cached;
|
| 200 |
+
}
|
| 201 |
+
const requestBody = buildPdfRequest(profile.opts, profile.meta || {});
|
| 202 |
+
const payload = {
|
| 203 |
+
profile: profile.name,
|
| 204 |
+
html: requestBody.html,
|
| 205 |
+
textOnlySizeMB: requestBody.textOnlySizeMB,
|
| 206 |
+
imageCount: requestBody.imageCount,
|
| 207 |
+
htmlSizeMB: Buffer.byteLength(requestBody.html, 'utf8') / (1024 * 1024),
|
| 208 |
+
requestBody: {
|
| 209 |
+
...requestBody,
|
| 210 |
+
html: undefined, // strip html from cache to keep file small
|
| 211 |
+
},
|
| 212 |
+
};
|
| 213 |
+
if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
|
| 214 |
+
fs.writeFileSync(file, JSON.stringify(payload));
|
| 215 |
+
return payload;
|
| 216 |
+
}
|
| 217 |
+
|
| 218 |
+
module.exports = { buildHtml, buildPdfRequest, prepareProfile, CODE_LINES };
|
| 219 |
+
|
| 220 |
+
if (require.main === module) {
|
| 221 |
+
const profiles = {
|
| 222 |
+
small: { name: 'small', opts: { textMB: 0.05, images: 0, codeRatio: 0.3 } },
|
| 223 |
+
medium: { name: 'medium', opts: { textMB: 0.3, images: 3, codeRatio: 0.35 } },
|
| 224 |
+
large: { name: 'large', opts: { textMB: 1.0, images: 10, codeRatio: 0.4 } },
|
| 225 |
+
};
|
| 226 |
+
for (const [key, prof] of Object.entries(profiles)) {
|
| 227 |
+
const p = prepareProfile(prof);
|
| 228 |
+
console.log(`${key.padEnd(8)} html=${p.htmlSizeMB.toFixed(3)} MB text=${p.textOnlySizeMB.toFixed(3)} MB images=${p.imageCount}`);
|
| 229 |
+
}
|
| 230 |
+
}
|
tests/stress/run-stress-test.js
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Stress test for the PDF export backend (/api/generate_pdf).
|
| 3 |
+
*
|
| 4 |
+
* Measures how many concurrent PDF exports the server can handle:
|
| 5 |
+
* - success rate
|
| 6 |
+
* - latency distribution (avg / p50 / p95 / max)
|
| 7 |
+
* - throughput (req/min)
|
| 8 |
+
*
|
| 9 |
+
* Usage:
|
| 10 |
+
* node stress/run-stress-test.js --base http://localhost:17861 --concurrency 1,2,4,6,8 --total 20 --profile medium
|
| 11 |
+
*
|
| 12 |
+
* Each concurrency level fires `total` requests in waves of `concurrency`
|
| 13 |
+
* parallel requests. Between levels the server is given a rest period
|
| 14 |
+
* (--settle-ms) so memory/CPU can drain back down.
|
| 15 |
+
*
|
| 16 |
+
* Output: JSON report written to stress/results/.
|
| 17 |
+
*/
|
| 18 |
+
|
| 19 |
+
const fs = require('fs');
|
| 20 |
+
const path = require('path');
|
| 21 |
+
const http = require('http');
|
| 22 |
+
const { prepareProfile } = require('./payload-generator');
|
| 23 |
+
|
| 24 |
+
function parseArgs(argv) {
|
| 25 |
+
const args = { concurrency: '1,2,4,6,8', total: 20, base: 'http://localhost:17861', profile: 'medium', settleMs: 10000, tag: '', timeoutMs: 180000 };
|
| 26 |
+
for (let i = 0; i < argv.length; i++) {
|
| 27 |
+
const m = argv[i].match(/^--([^=]+)(?:=(.*))?$/);
|
| 28 |
+
if (!m) continue;
|
| 29 |
+
if (m[2] !== undefined) {
|
| 30 |
+
args[m[1]] = m[2];
|
| 31 |
+
} else if (i + 1 < argv.length && !argv[i + 1].startsWith('--')) {
|
| 32 |
+
args[m[1]] = argv[++i];
|
| 33 |
+
} else {
|
| 34 |
+
args[m[1]] = true;
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
return args;
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
function percentile(sorted, p) {
|
| 41 |
+
if (sorted.length === 0) return 0;
|
| 42 |
+
const idx = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);
|
| 43 |
+
return sorted[idx];
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
function sendPdfRequest(baseUrl, requestBody, timeoutMs) {
|
| 47 |
+
const url = new URL('/api/generate_pdf', baseUrl);
|
| 48 |
+
const payload = JSON.stringify(requestBody);
|
| 49 |
+
return new Promise((resolve) => {
|
| 50 |
+
const started = Date.now();
|
| 51 |
+
const options = {
|
| 52 |
+
hostname: url.hostname,
|
| 53 |
+
port: url.port || 80,
|
| 54 |
+
path: url.pathname,
|
| 55 |
+
method: 'POST',
|
| 56 |
+
headers: {
|
| 57 |
+
'Content-Type': 'application/json',
|
| 58 |
+
'Content-Length': Buffer.byteLength(payload),
|
| 59 |
+
},
|
| 60 |
+
timeout: timeoutMs,
|
| 61 |
+
};
|
| 62 |
+
const req = http.request(options, (res) => {
|
| 63 |
+
const chunks = [];
|
| 64 |
+
res.on('data', (c) => chunks.push(c));
|
| 65 |
+
res.on('end', () => {
|
| 66 |
+
const elapsed = Date.now() - started;
|
| 67 |
+
const body = Buffer.concat(chunks);
|
| 68 |
+
if (res.statusCode === 200) {
|
| 69 |
+
resolve({ ok: true, status: 200, elapsed, pdfBytes: body.length });
|
| 70 |
+
} else {
|
| 71 |
+
resolve({ ok: false, status: res.statusCode, elapsed, error: body.toString().substring(0, 500) });
|
| 72 |
+
}
|
| 73 |
+
});
|
| 74 |
+
res.on('error', (e) => {
|
| 75 |
+
resolve({ ok: false, status: 0, elapsed: Date.now() - started, error: e.message });
|
| 76 |
+
});
|
| 77 |
+
});
|
| 78 |
+
req.on('error', (e) => {
|
| 79 |
+
resolve({ ok: false, status: 0, elapsed: Date.now() - started, error: e.message });
|
| 80 |
+
});
|
| 81 |
+
req.on('timeout', () => {
|
| 82 |
+
req.destroy(new Error(`timeout after ${timeoutMs}ms`));
|
| 83 |
+
});
|
| 84 |
+
req.write(payload);
|
| 85 |
+
req.end();
|
| 86 |
+
});
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
function fireWave(baseUrl, requestBody, concurrency, timeoutMs, onDone) {
|
| 90 |
+
return new Promise((resolve) => {
|
| 91 |
+
const results = [];
|
| 92 |
+
let active = 0;
|
| 93 |
+
let queued = concurrency;
|
| 94 |
+
let done = 0;
|
| 95 |
+
|
| 96 |
+
function startOne() {
|
| 97 |
+
active++;
|
| 98 |
+
sendPdfRequest(baseUrl, requestBody, timeoutMs).then((r) => {
|
| 99 |
+
results.push(r);
|
| 100 |
+
active--;
|
| 101 |
+
done++;
|
| 102 |
+
if (onDone) onDone(r);
|
| 103 |
+
if (queued > 0) { queued--; startOne(); }
|
| 104 |
+
else if (active === 0) resolve(results);
|
| 105 |
+
});
|
| 106 |
+
}
|
| 107 |
+
for (let i = 0; i < concurrency && queued > 0; i++) { queued--; startOne(); }
|
| 108 |
+
});
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
function sleep(ms) {
|
| 112 |
+
return new Promise((r) => setTimeout(r, ms));
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
async function runConcurrencyLevel(baseUrl, requestBody, concurrency, total, timeoutMs, settleMs, onProgress) {
|
| 116 |
+
const waves = Math.ceil(total / concurrency);
|
| 117 |
+
const all = [];
|
| 118 |
+
const startTs = Date.now();
|
| 119 |
+
for (let w = 0; w < waves; w++) {
|
| 120 |
+
const remaining = total - all.length;
|
| 121 |
+
const thisWave = Math.min(concurrency, remaining);
|
| 122 |
+
const waveResults = await fireWave(baseUrl, requestBody, thisWave, timeoutMs, onProgress);
|
| 123 |
+
all.push(...waveResults);
|
| 124 |
+
}
|
| 125 |
+
const durationMs = Date.now() - startTs;
|
| 126 |
+
await sleep(settleMs);
|
| 127 |
+
return analyze(all, durationMs);
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
function analyze(results, durationMs) {
|
| 131 |
+
const ok = results.filter((r) => r.ok);
|
| 132 |
+
const fail = results.filter((r) => !r.ok);
|
| 133 |
+
const latencies = ok.map((r) => r.elapsed).sort((a, b) => a - b);
|
| 134 |
+
const totalOk = ok.length;
|
| 135 |
+
const totalFail = fail.length;
|
| 136 |
+
// Wall-clock throughput: requests actually completed per minute across the level.
|
| 137 |
+
const rate = durationMs > 0 ? (totalOk / (durationMs / 60000)) : 0;
|
| 138 |
+
|
| 139 |
+
const errorBreakdown = {};
|
| 140 |
+
for (const f of fail) {
|
| 141 |
+
const key = f.status === 500 ? '500' : (f.status === 0 ? 'conn/' + (f.error || 'unknown') : String(f.status));
|
| 142 |
+
errorBreakdown[key] = (errorBreakdown[key] || 0) + 1;
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
return {
|
| 146 |
+
totalRequests: results.length,
|
| 147 |
+
success: totalOk,
|
| 148 |
+
failed: totalFail,
|
| 149 |
+
successRate: results.length ? (totalOk / results.length) : 0,
|
| 150 |
+
latencyMs: {
|
| 151 |
+
avg: latencies.length ? Math.round(latencies.reduce((a, b) => a + b, 0) / latencies.length) : 0,
|
| 152 |
+
p50: percentile(latencies, 50),
|
| 153 |
+
p90: percentile(latencies, 90),
|
| 154 |
+
p95: percentile(latencies, 95),
|
| 155 |
+
p99: percentile(latencies, 99),
|
| 156 |
+
max: latencies.length ? latencies[latencies.length - 1] : 0,
|
| 157 |
+
min: latencies.length ? latencies[0] : 0,
|
| 158 |
+
},
|
| 159 |
+
throughputPerMin: Math.round(rate),
|
| 160 |
+
errorBreakdown,
|
| 161 |
+
durationMs,
|
| 162 |
+
};
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
async function main() {
|
| 166 |
+
const args = parseArgs(process.argv.slice(2));
|
| 167 |
+
const baseUrl = args.base.replace(/\/$/, '');
|
| 168 |
+
const concurrencyLevels = args.concurrency.split(',').map((n) => parseInt(n, 10)).filter((n) => n > 0);
|
| 169 |
+
const total = parseInt(args.total, 10);
|
| 170 |
+
const timeoutMs = parseInt(args.timeoutMs, 10);
|
| 171 |
+
const settleMs = parseInt(args.settleMs, 10);
|
| 172 |
+
const profileName = args.profile;
|
| 173 |
+
|
| 174 |
+
const profileDefs = {
|
| 175 |
+
small: { name: 'small', opts: { textMB: 0.05, images: 0, codeRatio: 0.3 } },
|
| 176 |
+
medium: { name: 'medium', opts: { textMB: 0.3, images: 3, codeRatio: 0.35 } },
|
| 177 |
+
large: { name: 'large', opts: { textMB: 1.0, images: 10, codeRatio: 0.4 } },
|
| 178 |
+
};
|
| 179 |
+
const profileDef = profileDefs[profileName];
|
| 180 |
+
if (!profileDef) {
|
| 181 |
+
console.error(`Unknown profile: ${profileName}. Choose from ${Object.keys(profileDefs).join(', ')}`);
|
| 182 |
+
process.exit(1);
|
| 183 |
+
}
|
| 184 |
+
|
| 185 |
+
const profile = prepareProfile(profileDef);
|
| 186 |
+
const requestBody = { ...profile.requestBody, html: profile.html };
|
| 187 |
+
console.log(`Profile: ${profileName} html=${profile.htmlSizeMB.toFixed(3)} MB text=${profile.textOnlySizeMB.toFixed(3)} MB images=${profile.imageCount}`);
|
| 188 |
+
console.log(`Base URL: ${baseUrl}`);
|
| 189 |
+
console.log(`Concurrency levels: ${concurrencyLevels.join(', ')} total/level: ${total}\n`);
|
| 190 |
+
|
| 191 |
+
const report = {
|
| 192 |
+
generatedAt: new Date().toISOString(),
|
| 193 |
+
tag: args.tag,
|
| 194 |
+
baseUrl,
|
| 195 |
+
profile: { name: profileName, htmlSizeMB: profile.htmlSizeMB, textOnlySizeMB: profile.textOnlySizeMB, imageCount: profile.imageCount },
|
| 196 |
+
concurrencyLevels,
|
| 197 |
+
totalPerLevel: total,
|
| 198 |
+
levels: {},
|
| 199 |
+
};
|
| 200 |
+
|
| 201 |
+
const allRequests = [];
|
| 202 |
+
for (const conc of concurrencyLevels) {
|
| 203 |
+
process.stdout.write(`\n=== Concurrency ${conc} (${total} requests) ===\n`);
|
| 204 |
+
const level = await runConcurrencyLevel(baseUrl, requestBody, conc, total, timeoutMs, settleMs, (r) => {
|
| 205 |
+
allRequests.push(r);
|
| 206 |
+
process.stdout.write(r.ok ? '.' : 'X');
|
| 207 |
+
});
|
| 208 |
+
process.stdout.write('\n');
|
| 209 |
+
report.levels[String(conc)] = level;
|
| 210 |
+
const l = level;
|
| 211 |
+
console.log(` success=${l.success}/${l.totalRequests} (${(l.successRate * 100).toFixed(1)}%) ` +
|
| 212 |
+
`latency avg=${l.latencyMs.avg}ms p50=${l.latencyMs.p50}ms p95=${l.latencyMs.p95}ms max=${l.latencyMs.max}ms ` +
|
| 213 |
+
`throughput=${l.throughputPerMin}/min errors=${JSON.stringify(l.errorBreakdown)}`);
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
// Summary table
|
| 217 |
+
console.log('\n=== Summary ===');
|
| 218 |
+
console.log('Concurrency | Success | Rate% | avg(ms) | p50(ms) | p95(ms) | max(ms) | req/min |');
|
| 219 |
+
console.log('------------|---------|--------|---------|---------|---------|---------|---------|');
|
| 220 |
+
for (const conc of concurrencyLevels) {
|
| 221 |
+
const l = report.levels[String(conc)];
|
| 222 |
+
console.log(`${String(conc).padEnd(11)}| ${String(l.success).padEnd(8)}| ${(l.successRate * 100).toFixed(1).padEnd(7)}| ${String(l.latencyMs.avg).padEnd(8)}| ${String(l.latencyMs.p50).padEnd(8)}| ${String(l.latencyMs.p95).padEnd(8)}| ${String(l.latencyMs.max).padEnd(8)}| ${String(l.throughputPerMin).padEnd(8)}|`);
|
| 223 |
+
}
|
| 224 |
+
|
| 225 |
+
const stamp = new Date().toISOString().replace(/[:T]/g, '-').slice(0, 19);
|
| 226 |
+
const tagSuffix = args.tag ? `-${args.tag}` : '';
|
| 227 |
+
const outFile = path.join(__dirname, 'results', `stress-${profileName}${tagSuffix}-${stamp}.json`);
|
| 228 |
+
fs.writeFileSync(outFile, JSON.stringify(report, null, 2));
|
| 229 |
+
console.log(`\nReport saved: ${outFile}`);
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
main().catch((e) => {
|
| 233 |
+
console.error(e);
|
| 234 |
+
process.exit(1);
|
| 235 |
+
});
|
tests/stress/test-widget-batch.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Batch widget render verification — mimics a real PDF/DOCX export's
|
| 3 |
+
* _renderWidgets() call to /api/render_charts with several widgets.
|
| 4 |
+
* Confirms the shared singleton browser handles concurrent pages correctly.
|
| 5 |
+
*/
|
| 6 |
+
const http = require('http');
|
| 7 |
+
|
| 8 |
+
const BASE = process.env.BASE_URL || 'http://localhost:17861';
|
| 9 |
+
|
| 10 |
+
function chartWidget(title, type, labels) {
|
| 11 |
+
return {
|
| 12 |
+
type: 'chart',
|
| 13 |
+
title,
|
| 14 |
+
html: `<div style="width:650px;height:300px;"><canvas id="c"></canvas></div>
|
| 15 |
+
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
|
| 16 |
+
<script>new Chart(document.getElementById("c"),{type:"${type}",data:{labels:${JSON.stringify(labels)},datasets:[{data:[3,5,2,4]}]},options:{animation:false}});</script>`
|
| 17 |
+
};
|
| 18 |
+
}
|
| 19 |
+
|
| 20 |
+
function mermaidWidget(title, def) {
|
| 21 |
+
return {
|
| 22 |
+
type: 'mermaid',
|
| 23 |
+
title,
|
| 24 |
+
html: `<div class="mermaid">${def}</div>
|
| 25 |
+
<script type="module">
|
| 26 |
+
import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.esm.min.mjs';
|
| 27 |
+
mermaid.initialize({ startOnLoad: true, theme: 'default' });
|
| 28 |
+
</script>`
|
| 29 |
+
};
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
const widgets = [
|
| 33 |
+
chartWidget('bar', 'bar', ['A', 'B', 'C']),
|
| 34 |
+
chartWidget('line', 'line', ['1', '2', '3']),
|
| 35 |
+
chartWidget('pie', 'pie', ['X', 'Y', 'Z']),
|
| 36 |
+
mermaidWidget('flow', 'graph TD\n A[Start] --> B{Check}\n B -->|Yes| C[Go]\n B -->|No| D[Stop]'),
|
| 37 |
+
chartWidget('doughnut', 'doughnut', ['P', 'Q']),
|
| 38 |
+
];
|
| 39 |
+
|
| 40 |
+
const body = JSON.stringify({ widgets, theme: 'light' });
|
| 41 |
+
const started = Date.now();
|
| 42 |
+
|
| 43 |
+
const req = http.request({
|
| 44 |
+
hostname: 'localhost',
|
| 45 |
+
port: 17861,
|
| 46 |
+
path: '/api/render_charts',
|
| 47 |
+
method: 'POST',
|
| 48 |
+
headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
|
| 49 |
+
timeout: 180000,
|
| 50 |
+
}, (res) => {
|
| 51 |
+
const chunks = [];
|
| 52 |
+
res.on('data', (c) => chunks.push(c));
|
| 53 |
+
res.on('end', () => {
|
| 54 |
+
const elapsed = ((Date.now() - started) / 1000).toFixed(2);
|
| 55 |
+
const data = JSON.parse(Buffer.concat(chunks).toString());
|
| 56 |
+
const results = data.results || [];
|
| 57 |
+
const ok = results.filter((r) => r.success).length;
|
| 58 |
+
console.log(`HTTP ${res.statusCode} in ${elapsed}s: ${ok}/${results.length} widgets OK`);
|
| 59 |
+
for (const r of results) {
|
| 60 |
+
console.log(` [${r.index}] ${r.type}/${r.title}: ${r.success ? 'OK len=' + r.dataUrl.length : 'FAIL ' + (r.error || '')}`);
|
| 61 |
+
}
|
| 62 |
+
});
|
| 63 |
+
});
|
| 64 |
+
req.on('error', (e) => { console.error('ERR', e.message); process.exit(1); });
|
| 65 |
+
req.write(body);
|
| 66 |
+
req.end();
|
tests/stress/verify-pdf-output.js
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Generate a PDF via the backend and save to disk for quality verification.
|
| 3 |
+
* Usage: node tests/stress/verify-pdf-output.js <output.pdf>
|
| 4 |
+
*/
|
| 5 |
+
const fs = require('fs');
|
| 6 |
+
const http = require('http');
|
| 7 |
+
const path = require('path');
|
| 8 |
+
const { prepareProfile } = require('./payload-generator');
|
| 9 |
+
|
| 10 |
+
const BASE = process.env.BASE_URL || 'http://localhost:17861';
|
| 11 |
+
const outFile = process.argv[2] || path.join(__dirname, 'results', 'verify-output.pdf');
|
| 12 |
+
|
| 13 |
+
async function main() {
|
| 14 |
+
const profile = prepareProfile({ name: 'medium', opts: { textMB: 0.3, images: 3, codeRatio: 0.35 } });
|
| 15 |
+
const requestBody = { ...profile.requestBody, html: profile.html };
|
| 16 |
+
const payload = JSON.stringify(requestBody);
|
| 17 |
+
const url = new URL('/api/generate_pdf', BASE);
|
| 18 |
+
|
| 19 |
+
const started = Date.now();
|
| 20 |
+
const res = await new Promise((resolve, reject) => {
|
| 21 |
+
const req = http.request({ hostname: url.hostname, port: url.port || 80, path: url.pathname, method: 'POST',
|
| 22 |
+
headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(payload) }, timeout: 120000 }, resolve);
|
| 23 |
+
req.on('error', reject);
|
| 24 |
+
req.write(payload);
|
| 25 |
+
req.end();
|
| 26 |
+
});
|
| 27 |
+
|
| 28 |
+
const chunks = [];
|
| 29 |
+
for await (const c of res) chunks.push(c);
|
| 30 |
+
const buf = Buffer.concat(chunks);
|
| 31 |
+
const elapsed = ((Date.now() - started) / 1000).toFixed(2);
|
| 32 |
+
console.log(`HTTP ${res.statusCode} in ${elapsed}s, bytes=${buf.length}`);
|
| 33 |
+
|
| 34 |
+
if (res.statusCode !== 200) {
|
| 35 |
+
console.log(buf.toString().substring(0, 500));
|
| 36 |
+
process.exit(1);
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
fs.writeFileSync(outFile, buf);
|
| 40 |
+
const head = buf.slice(0, 8).toString('latin1');
|
| 41 |
+
console.log(`Saved: ${outFile}`);
|
| 42 |
+
console.log(`Header magic: ${head} (expects %PDF-1.x)`);
|
| 43 |
+
const pageCount = (buf.toString('latin1').match(/\/Type\s*\/Page[^s]/g) || []).length;
|
| 44 |
+
console.log(`Approx page count: ${pageCount}`);
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
main().catch((e) => { console.error(e); process.exit(1); });
|
tools/compare-dirs.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* 递归对比两个目录,列出差异文件(排除 node_modules/.git/dist 等)。
|
| 3 |
+
* 用法: node compare-dirs.js <dirA> <dirB>
|
| 4 |
+
*/
|
| 5 |
+
const fs = require('fs');
|
| 6 |
+
const path = require('path');
|
| 7 |
+
const crypto = require('crypto');
|
| 8 |
+
|
| 9 |
+
const EXCLUDE_DIRS = new Set(['node_modules', '.git', 'temp', 'test-results', 'make', 'dist', 'tools', 'tests', 'stress', 'hf-trend', '.sisyphus', '.opencode', 'output', 'test-outputs']);
|
| 10 |
+
const EXCLUDE_FILES = new Set(['PERFORMANCE_ANALYSIS.md', 'PERFORMANCE_STRESS_TEST_2026-08-03.md', 'HF_TOKEN.md']);
|
| 11 |
+
|
| 12 |
+
const [A, B] = process.argv.slice(2);
|
| 13 |
+
if (!A || !B) { console.error('need two dirs'); process.exit(1); }
|
| 14 |
+
|
| 15 |
+
const md5 = (p) => crypto.createHash('md5').update(fs.readFileSync(p)).digest('hex');
|
| 16 |
+
|
| 17 |
+
function walk(d, base) {
|
| 18 |
+
const out = new Map();
|
| 19 |
+
if (!fs.existsSync(d)) return out;
|
| 20 |
+
for (const f of fs.readdirSync(d)) {
|
| 21 |
+
const fp = path.join(d, f);
|
| 22 |
+
if (fs.statSync(fp).isDirectory()) {
|
| 23 |
+
if (EXCLUDE_DIRS.has(f)) continue;
|
| 24 |
+
for (const [rel, h] of walk(fp, base)) out.set(rel, h);
|
| 25 |
+
} else {
|
| 26 |
+
if (EXCLUDE_FILES.has(f)) continue;
|
| 27 |
+
const rel = path.relative(base, fp).replace(/\\/g, '/');
|
| 28 |
+
out.set(rel, md5(fp));
|
| 29 |
+
}
|
| 30 |
+
}
|
| 31 |
+
return out;
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
const mapA = walk(A, A);
|
| 35 |
+
const mapB = walk(B, B);
|
| 36 |
+
|
| 37 |
+
const all = new Set([...mapA.keys(), ...mapB.keys()]);
|
| 38 |
+
const changed = [];
|
| 39 |
+
const onlyA = [];
|
| 40 |
+
const onlyB = [];
|
| 41 |
+
for (const rel of all) {
|
| 42 |
+
const ha = mapA.get(rel);
|
| 43 |
+
const hb = mapB.get(rel);
|
| 44 |
+
if (ha && hb) { if (ha !== hb) changed.push(rel); }
|
| 45 |
+
else if (ha && !hb) onlyA.push(rel);
|
| 46 |
+
else onlyB.push(rel);
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
changed.sort(); onlyA.sort(); onlyB.sort();
|
| 50 |
+
console.log(`=== A=${A}`);
|
| 51 |
+
console.log(`=== B=${B}`);
|
| 52 |
+
console.log(`\n[不同内容] ${changed.length} 个:`);
|
| 53 |
+
for (const f of changed) console.log(` M ${f}`);
|
| 54 |
+
console.log(`\n[仅 A 有] ${onlyA.length} 个:`);
|
| 55 |
+
for (const f of onlyA) console.log(` A ${f}`);
|
| 56 |
+
console.log(`\n[仅 B 有] ${onlyB.length} 个:`);
|
| 57 |
+
for (const f of onlyB) console.log(` B ${f}`);
|