Spaces:
Runtime error
✨ feat(wasabi): add hls video streaming and s3 proxy
Browse files- implement full hls video streaming architecture with wasabi s3
- server acts as proxy cdn to stream hls content from wasabi
- add api endpoints for presigned urls, file streaming, hls upload and deletion
- integrate hls conversion service for adaptive hls generation
- introduce filetype enum for better file categorization
- configure wasabi s3 credentials via environment variables
📝 docs(wasabi): document hls streaming architecture
- create hls-streaming-architecture.md with detailed explanations of:
- upload and hls conversion flow
- streaming flow (proxy cdn magic) and url rewriting
- performance analysis for local vs production deployment
- architecture benefits and debugging tips
- add mermaid-diagram.md for a visual representation of the hls streaming process
- .env.example +7 -0
- src/shared/modules/upload/upload.module.ts +2 -0
- src/shared/modules/upload/wasabi-storage/docs/HLS-STREAMING-ARCHITECTURE.md +860 -0
- src/shared/modules/upload/wasabi-storage/docs/MERMAID-DIAGRAM.md +513 -0
- src/shared/modules/upload/wasabi-storage/enums/file-type.enum.ts +5 -0
- src/shared/modules/upload/wasabi-storage/wasabi-storage.config.ts +9 -0
- src/shared/modules/upload/wasabi-storage/wasabi-storage.controller.ts +252 -0
- src/shared/modules/upload/wasabi-storage/wasabi-storage.module.ts +14 -0
- src/shared/modules/upload/wasabi-storage/wasabi-storage.service.ts +624 -0
|
@@ -146,6 +146,13 @@ IDRIVE_E2_SECRET_KEY="your-idrive-secret-key"
|
|
| 146 |
IDRIVE_E2_BUCKET="streamflix"
|
| 147 |
IDRIVE_E2_CDN_URL="https://idrive-e2-ap-southeast.aksharbhesaniya.dev"
|
| 148 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
# Dropbox Storage Credentials
|
| 150 |
DROPBOX_ACCESS_TOKEN="your-dropbox-access-token"
|
| 151 |
DROPBOX_APP_FOLDER="/StreamflixDev"
|
|
|
|
| 146 |
IDRIVE_E2_BUCKET="streamflix"
|
| 147 |
IDRIVE_E2_CDN_URL="https://idrive-e2-ap-southeast.aksharbhesaniya.dev"
|
| 148 |
|
| 149 |
+
# Wasabi Storage Credentials
|
| 150 |
+
WASABI_ENDPOINT="https://s3.us-west-1.wasabisys.com"
|
| 151 |
+
WASABI_REGION="us-west-1"
|
| 152 |
+
WASABI_ACCESS_KEY_ID="your-wasabi-access-key"
|
| 153 |
+
WASABI_SECRET_ACCESS_KEY="your-wasabi-secret-key"
|
| 154 |
+
WASABI_BUCKET_NAME="streamflix-backed"
|
| 155 |
+
|
| 156 |
# Dropbox Storage Credentials
|
| 157 |
DROPBOX_ACCESS_TOKEN="your-dropbox-access-token"
|
| 158 |
DROPBOX_APP_FOLDER="/StreamflixDev"
|
|
@@ -14,6 +14,7 @@ import { DropboxStorageModule } from './dropbox-storage/dropbox-storage.module';
|
|
| 14 |
import { GoogleDriveModule } from './google-drive/google-drive.module';
|
| 15 |
import { BoxStorageModule } from './box-storage/box-storage.module';
|
| 16 |
import { UploadcareModule } from './uploadcare/uploadcare.module';
|
|
|
|
| 17 |
|
| 18 |
@Module({
|
| 19 |
imports: [
|
|
@@ -32,6 +33,7 @@ import { UploadcareModule } from './uploadcare/uploadcare.module';
|
|
| 32 |
GoogleDriveModule,
|
| 33 |
BoxStorageModule,
|
| 34 |
UploadcareModule,
|
|
|
|
| 35 |
],
|
| 36 |
})
|
| 37 |
export class UploadModule {}
|
|
|
|
| 14 |
import { GoogleDriveModule } from './google-drive/google-drive.module';
|
| 15 |
import { BoxStorageModule } from './box-storage/box-storage.module';
|
| 16 |
import { UploadcareModule } from './uploadcare/uploadcare.module';
|
| 17 |
+
import { WasabiStorageModule } from './wasabi-storage/wasabi-storage.module';
|
| 18 |
|
| 19 |
@Module({
|
| 20 |
imports: [
|
|
|
|
| 33 |
GoogleDriveModule,
|
| 34 |
BoxStorageModule,
|
| 35 |
UploadcareModule,
|
| 36 |
+
WasabiStorageModule,
|
| 37 |
],
|
| 38 |
})
|
| 39 |
export class UploadModule {}
|
|
@@ -0,0 +1,860 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 🎬 Wasabi HLS Streaming Architecture
|
| 2 |
+
|
| 3 |
+
Complete guide to how HLS video streaming works with Wasabi storage and proxy CDN architecture.
|
| 4 |
+
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
## 📋 Table of Contents
|
| 8 |
+
|
| 9 |
+
1. [Overview](#overview)
|
| 10 |
+
2. [Upload & Conversion Flow](#upload--conversion-flow)
|
| 11 |
+
3. [Streaming Flow](#streaming-flow)
|
| 12 |
+
4. [URL Rewriting Flow](#url-rewriting-flow)
|
| 13 |
+
5. [Why Local Testing is Slow](#why-local-testing-is-slow)
|
| 14 |
+
6. [Production Performance](#production-performance)
|
| 15 |
+
|
| 16 |
+
---
|
| 17 |
+
|
| 18 |
+
## 🎯 Overview
|
| 19 |
+
|
| 20 |
+
**Problem:** Wasabi free trial doesn't allow public bucket access
|
| 21 |
+
**Solution:** Server acts as proxy CDN - streams files from Wasabi to clients
|
| 22 |
+
**Benefit:** No direct Wasabi URLs exposed, full control over streaming
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## 📤 Upload & Conversion Flow
|
| 27 |
+
|
| 28 |
+
```
|
| 29 |
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
| 30 |
+
│ UPLOAD & HLS CONVERSION FLOW │
|
| 31 |
+
└─────────────────────────────────────────────────────────────────────────────┘
|
| 32 |
+
|
| 33 |
+
Step 1: User uploads video file
|
| 34 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 35 |
+
┌──────────┐
|
| 36 |
+
│ Client │ POST /wasabi/upload-video-hls
|
| 37 |
+
│ (Browser)│ Content-Type: multipart/form-data
|
| 38 |
+
└─────┬────┘ { video: video.mp4 }
|
| 39 |
+
│
|
| 40 |
+
│ HTTP Upload
|
| 41 |
+
▼
|
| 42 |
+
┌─────────────────────────────────────┐
|
| 43 |
+
│ WasabiStorageController │
|
| 44 |
+
│ @Post('upload-video-hls') │
|
| 45 |
+
│ │
|
| 46 |
+
│ • Receives file via multer │
|
| 47 |
+
│ • Saves to: ./resources/temp/ │
|
| 48 |
+
│ • Generates uploadId │
|
| 49 |
+
│ (e.g., 1766933325669-42b7k6) │
|
| 50 |
+
└──────────────┬──────────────────────┘
|
| 51 |
+
│
|
| 52 |
+
│ Call service
|
| 53 |
+
▼
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
Step 2: Async HLS Conversion (FFmpeg)
|
| 57 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 58 |
+
┌─────────────────────────────────────┐
|
| 59 |
+
│ WasabiStorageService │
|
| 60 |
+
│ startVideoProcessing() │
|
| 61 |
+
│ │
|
| 62 |
+
│ Returns uploadId immediately │
|
| 63 |
+
│ (non-blocking, async processing) │
|
| 64 |
+
└──────────────┬──────────────────────┘
|
| 65 |
+
│
|
| 66 |
+
│ Async processing in background
|
| 67 |
+
▼
|
| 68 |
+
┌─────────────────────────────────────┐
|
| 69 |
+
│ HlsConversionService │
|
| 70 |
+
│ convertVideoToAdaptiveHLS() │
|
| 71 |
+
│ │
|
| 72 |
+
│ Converts video to HLS format: │
|
| 73 |
+
│ • 1080p (1920x1080, 5500k bitrate) │
|
| 74 |
+
│ • 720p (1280x720, 3000k bitrate) │
|
| 75 |
+
│ • 480p (854x480, 1200k bitrate) │
|
| 76 |
+
│ • Audio-only (128k bitrate) │
|
| 77 |
+
│ │
|
| 78 |
+
│ Generates files: │
|
| 79 |
+
│ • master.m3u8 (main playlist) │
|
| 80 |
+
│ • index_1080p.m3u8 │
|
| 81 |
+
│ • index_720p.m3u8 │
|
| 82 |
+
│ • index_480p.m3u8 │
|
| 83 |
+
│ • index_audio.m3u8 │
|
| 84 |
+
│ • segment_1080p_000.ts to XXX.ts │
|
| 85 |
+
│ • segment_720p_000.ts to XXX.ts │
|
| 86 |
+
│ • segment_480p_000.ts to XXX.ts │
|
| 87 |
+
│ • segment_audio_000.ts to XXX.ts │
|
| 88 |
+
│ │
|
| 89 |
+
│ Saved to: /tmp/streamflix-hls/ │
|
| 90 |
+
│ {uploadId}/ │
|
| 91 |
+
└──────────────┬──────────────────────┘
|
| 92 |
+
│
|
| 93 |
+
│ Conversion complete
|
| 94 |
+
▼
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
Step 3: Upload all files to Wasabi
|
| 98 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 99 |
+
┌─────────────────────────────────────┐
|
| 100 |
+
│ WasabiStorageService │
|
| 101 |
+
│ uploadHLSFilesToWasabi() │
|
| 102 |
+
│ │
|
| 103 |
+
│ Uploads to Wasabi S3: │
|
| 104 |
+
│ • hls/{uploadId}/master.m3u8 │
|
| 105 |
+
│ • hls/{uploadId}/index_1080p.m3u8 │
|
| 106 |
+
│ • hls/{uploadId}/index_720p.m3u8 │
|
| 107 |
+
│ • hls/{uploadId}/index_480p.m3u8 │
|
| 108 |
+
│ • hls/{uploadId}/index_audio.m3u8 │
|
| 109 |
+
│ • hls/{uploadId}/segment_*.ts │
|
| 110 |
+
│ (all segments for all qualities) │
|
| 111 |
+
└──────────────┬──────────────────────┘
|
| 112 |
+
│
|
| 113 |
+
│ Upload complete
|
| 114 |
+
▼
|
| 115 |
+
┌─────────────────────────────────────┐
|
| 116 |
+
│ Wasabi S3 Storage │
|
| 117 |
+
│ Bucket: streamflix-backed │
|
| 118 |
+
│ Region: us-west-1 │
|
| 119 |
+
│ │
|
| 120 |
+
│ Storage structure: │
|
| 121 |
+
│ hls/ │
|
| 122 |
+
│ └── 1766933325669-42b7k6/ │
|
| 123 |
+
│ ├── master.m3u8 │
|
| 124 |
+
│ ├── index_1080p.m3u8 │
|
| 125 |
+
│ ├── index_720p.m3u8 │
|
| 126 |
+
│ ├── index_480p.m3u8 │
|
| 127 |
+
│ ├── index_audio.m3u8 │
|
| 128 |
+
│ ├── segment_1080p_000.ts │
|
| 129 |
+
│ ├── segment_1080p_001.ts │
|
| 130 |
+
│ ├── ... (hundreds of segments) │
|
| 131 |
+
│ ├── segment_720p_000.ts │
|
| 132 |
+
│ ├── segment_480p_000.ts │
|
| 133 |
+
│ └── segment_audio_000.ts │
|
| 134 |
+
└─────────────────────────────────────┘
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
Step 4: Cleanup local files
|
| 138 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 139 |
+
┌─────────────────────────────────────┐
|
| 140 |
+
│ HlsConversionService │
|
| 141 |
+
│ cleanupFiles(uploadId) │
|
| 142 |
+
│ │
|
| 143 |
+
│ Deletes from local disk: │
|
| 144 |
+
│ • /tmp/streamflix-hls/{uploadId}/ │
|
| 145 |
+
│ • ./resources/temp/video-*.mp4 │
|
| 146 |
+
│ │
|
| 147 |
+
│ Result: Files only in Wasabi │
|
| 148 |
+
└─────────────────────────────────────┘
|
| 149 |
+
```
|
| 150 |
+
|
| 151 |
+
---
|
| 152 |
+
|
| 153 |
+
## 📥 Streaming Flow (The Proxy CDN Magic)
|
| 154 |
+
|
| 155 |
+
```
|
| 156 |
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
| 157 |
+
│ HLS STREAMING FLOW (PROXY CDN) │
|
| 158 |
+
└─────────────────────────────────────────────────────────────────────────────┘
|
| 159 |
+
|
| 160 |
+
Step 1: User requests master playlist
|
| 161 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 162 |
+
┌──────────┐
|
| 163 |
+
│ Client │ GET /wasabi/hls?uploadId=1766933325669-42b7k6
|
| 164 |
+
│ (HLS │
|
| 165 |
+
│ Player) │
|
| 166 |
+
└─────┬────┘
|
| 167 |
+
│
|
| 168 |
+
│ HTTP Request
|
| 169 |
+
▼
|
| 170 |
+
┌─────────────────────────────────────┐
|
| 171 |
+
│ WasabiStorageController │
|
| 172 |
+
│ @Get('hls') │
|
| 173 |
+
│ @SkipApiToken() (public) │
|
| 174 |
+
└──────────────┬──────────────────────┘
|
| 175 |
+
│
|
| 176 |
+
│ Call service
|
| 177 |
+
▼
|
| 178 |
+
┌─────────────────────────────────────┐
|
| 179 |
+
│ WasabiStorageService │
|
| 180 |
+
│ getRewrittenM3U8(uploadId) │
|
| 181 |
+
│ │
|
| 182 |
+
│ 1. Fetch from Wasabi: │
|
| 183 |
+
│ hls/{uploadId}/master.m3u8 │
|
| 184 |
+
└──────────────┬──────────────────────┘
|
| 185 |
+
│
|
| 186 |
+
│ Internal S3 request (presigned URL)
|
| 187 |
+
▼
|
| 188 |
+
┌─────────────────────────────────────┐
|
| 189 |
+
│ Wasabi S3 Storage │
|
| 190 |
+
│ Returns master.m3u8 content: │
|
| 191 |
+
│ │
|
| 192 |
+
│ #EXTM3U │
|
| 193 |
+
│ #EXT-X-MEDIA:URI="index_audio.m3u8"│
|
| 194 |
+
│ #EXT-X-STREAM-INF:BANDWIDTH=1328000│
|
| 195 |
+
│ index_480p.m3u8 │
|
| 196 |
+
│ #EXT-X-STREAM-INF:BANDWIDTH=3128000│
|
| 197 |
+
│ index_720p.m3u8 │
|
| 198 |
+
│ #EXT-X-STREAM-INF:BANDWIDTH=5628000│
|
| 199 |
+
│ index_1080p.m3u8 │
|
| 200 |
+
└──────────────┬──────────────────────┘
|
| 201 |
+
│
|
| 202 |
+
│ Stream master.m3u8 content
|
| 203 |
+
▼
|
| 204 |
+
┌─────────────────────────────────────┐
|
| 205 |
+
│ WasabiStorageService │
|
| 206 |
+
│ getRewrittenM3U8() │
|
| 207 |
+
│ │
|
| 208 |
+
│ 2. Rewrite URLs (URL MAGIC!) │
|
| 209 |
+
│ │
|
| 210 |
+
│ Before: │
|
| 211 |
+
│ URI="index_audio.m3u8" │
|
| 212 |
+
│ index_480p.m3u8 │
|
| 213 |
+
│ index_720p.m3u8 │
|
| 214 |
+
│ │
|
| 215 |
+
│ After: │
|
| 216 |
+
│ URI="https://domain/wasabi/ │
|
| 217 |
+
│ file?path=hls%2F...%2F │
|
| 218 |
+
│ index_audio.m3u8" │
|
| 219 |
+
│ https://domain/wasabi/ │
|
| 220 |
+
│ file?path=hls%2F...%2F │
|
| 221 |
+
│ index_480p.m3u8 │
|
| 222 |
+
│ https://domain/wasabi/ │
|
| 223 |
+
│ file?path=hls%2F...%2F │
|
| 224 |
+
│ index_720p.m3u8 │
|
| 225 |
+
└──────────────┬──────────────────────┘
|
| 226 |
+
│
|
| 227 |
+
│ Return rewritten playlist
|
| 228 |
+
▼
|
| 229 |
+
┌──────────┐
|
| 230 |
+
│ Client │ Receives master.m3u8 with
|
| 231 |
+
│ (HLS │ absolute URLs to server proxy
|
| 232 |
+
│ Player) │
|
| 233 |
+
└─────┬────┘
|
| 234 |
+
│
|
| 235 |
+
│ Player parses, selects quality (e.g., 720p)
|
| 236 |
+
▼
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
Step 2: Player requests quality playlist
|
| 240 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 241 |
+
┌──────────┐
|
| 242 |
+
│ Client │ GET /wasabi/file?path=hls%2F...%2Findex_720p.m3u8
|
| 243 |
+
└─────┬────┘
|
| 244 |
+
│
|
| 245 |
+
│ HTTP Request
|
| 246 |
+
▼
|
| 247 |
+
┌─────────────────────────────────────┐
|
| 248 |
+
│ WasabiStorageController │
|
| 249 |
+
│ @Get('file') │
|
| 250 |
+
│ @SkipApiToken() (public) │
|
| 251 |
+
│ │
|
| 252 |
+
│ Detects: .m3u8 file in hls/ folder │
|
| 253 |
+
│ Triggers: Automatic URL rewriting │
|
| 254 |
+
└──────────────┬──────────────────────┘
|
| 255 |
+
│
|
| 256 |
+
│ Call service
|
| 257 |
+
▼
|
| 258 |
+
┌─────────────────────────────────────┐
|
| 259 |
+
│ WasabiStorageService │
|
| 260 |
+
│ streamFile(path) │
|
| 261 |
+
│ │
|
| 262 |
+
│ 1. Fetch from Wasabi: │
|
| 263 |
+
│ hls/{uploadId}/index_720p.m3u8 │
|
| 264 |
+
└──────────────┬──────────────────────┘
|
| 265 |
+
│
|
| 266 |
+
│ Internal S3 request
|
| 267 |
+
▼
|
| 268 |
+
┌─────────────────────────────────────┐
|
| 269 |
+
│ Wasabi S3 Storage │
|
| 270 |
+
│ Returns index_720p.m3u8: │
|
| 271 |
+
│ │
|
| 272 |
+
│ #EXTM3U │
|
| 273 |
+
│ #EXTINF:6.000000, │
|
| 274 |
+
│ segment_720p_000.ts │
|
| 275 |
+
│ #EXTINF:6.000000, │
|
| 276 |
+
│ segment_720p_001.ts │
|
| 277 |
+
│ #EXTINF:6.000000, │
|
| 278 |
+
│ segment_720p_002.ts │
|
| 279 |
+
│ ... (more segments) │
|
| 280 |
+
└──────────────┬──────────────────────┘
|
| 281 |
+
│
|
| 282 |
+
│ Stream playlist content
|
| 283 |
+
▼
|
| 284 |
+
┌─────────────────────────────────────┐
|
| 285 |
+
│ WasabiStorageService │
|
| 286 |
+
│ streamFile() - Auto Rewrite! │
|
| 287 |
+
│ │
|
| 288 |
+
│ 2. Rewrite segment URLs │
|
| 289 |
+
│ │
|
| 290 |
+
│ Before: │
|
| 291 |
+
│ segment_720p_000.ts │
|
| 292 |
+
│ segment_720p_001.ts │
|
| 293 |
+
│ segment_720p_002.ts │
|
| 294 |
+
│ │
|
| 295 |
+
│ After: │
|
| 296 |
+
│ https://domain/wasabi/ │
|
| 297 |
+
│ file?path=hls%2F...%2F │
|
| 298 |
+
│ segment_720p_000.ts │
|
| 299 |
+
│ https://domain/wasabi/ │
|
| 300 |
+
│ file?path=hls%2F...%2F │
|
| 301 |
+
│ segment_720p_001.ts │
|
| 302 |
+
│ https://domain/wasabi/ │
|
| 303 |
+
│ file?path=hls%2F...%2F │
|
| 304 |
+
│ segment_720p_002.ts │
|
| 305 |
+
└──────────────┬──────────────────────┘
|
| 306 |
+
│
|
| 307 |
+
│ Return rewritten playlist
|
| 308 |
+
▼
|
| 309 |
+
┌──────────┐
|
| 310 |
+
│ Client │ Receives quality playlist with
|
| 311 |
+
│ (HLS │ absolute URLs to server proxy
|
| 312 |
+
│ Player) │
|
| 313 |
+
└─────┬────┘
|
| 314 |
+
│
|
| 315 |
+
│ Player starts downloading segments
|
| 316 |
+
▼
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
Step 3: Player requests video segments (MANY PARALLEL REQUESTS)
|
| 320 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 321 |
+
┌──────────┐
|
| 322 |
+
│ Client │ GET /wasabi/file?path=hls%2F...%2Fsegment_720p_000.ts
|
| 323 |
+
│ (HLS │ GET /wasabi/file?path=hls%2F...%2Fsegment_720p_001.ts
|
| 324 |
+
│ Player) │ GET /wasabi/file?path=hls%2F...%2Fsegment_720p_002.ts
|
| 325 |
+
└─────┬────┘ (Multiple parallel requests)
|
| 326 |
+
│
|
| 327 |
+
│ HTTP Requests (parallel)
|
| 328 |
+
▼
|
| 329 |
+
┌─────────────────────────────────────┐
|
| 330 |
+
│ WasabiStorageController │
|
| 331 |
+
│ @Get('file') │
|
| 332 |
+
│ @SkipApiToken() (public) │
|
| 333 |
+
│ │
|
| 334 |
+
│ Detects: .ts file (video segment) │
|
| 335 |
+
│ Action: Direct stream (no rewrite) │
|
| 336 |
+
└──────────────┬──────────────────────┘
|
| 337 |
+
│
|
| 338 |
+
│ Call service
|
| 339 |
+
▼
|
| 340 |
+
┌─────────────────────────────────────┐
|
| 341 |
+
│ WasabiStorageService │
|
| 342 |
+
│ streamFile(path) │
|
| 343 |
+
│ │
|
| 344 |
+
│ Fetch from Wasabi: │
|
| 345 |
+
│ hls/{uploadId}/segment_720p_000.ts │
|
| 346 |
+
└──────────────┬──────────────────────┘
|
| 347 |
+
│
|
| 348 |
+
│ Internal S3 request
|
| 349 |
+
▼
|
| 350 |
+
┌─────────────────────────────────────┐
|
| 351 |
+
│ Wasabi S3 Storage │
|
| 352 |
+
│ Returns video segment: │
|
| 353 |
+
│ Content-Type: video/mp2t │
|
| 354 |
+
│ Size: ~1-2 MB (6 seconds of video) │
|
| 355 |
+
└──────────────┬──────────────────────┘
|
| 356 |
+
│
|
| 357 |
+
│ Stream segment (DIRECT PIPE)
|
| 358 |
+
▼
|
| 359 |
+
┌─────────────────────────────────────┐
|
| 360 |
+
│ WasabiStorageService │
|
| 361 |
+
│ streamFile() │
|
| 362 |
+
│ │
|
| 363 |
+
│ ⚡ CRITICAL: Direct piping! │
|
| 364 |
+
│ result.stream.pipe(res) │
|
| 365 |
+
│ │
|
| 366 |
+
│ • NO buffering on server │
|
| 367 |
+
│ • Chunks stream immediately │
|
| 368 |
+
│ • Server memory stays low │
|
| 369 |
+
└──────────────┬──────────────────────┘
|
| 370 |
+
│
|
| 371 |
+
│ Pipe stream chunks
|
| 372 |
+
▼
|
| 373 |
+
┌──────────┐
|
| 374 |
+
│ Client │ Receives video chunks in real-time
|
| 375 |
+
│ (HLS │ Player buffers and plays video
|
| 376 |
+
│ Player) │
|
| 377 |
+
└──────────┘
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
🔄 This repeats for EVERY segment (could be 100+ segments per quality)
|
| 381 |
+
```
|
| 382 |
+
|
| 383 |
+
---
|
| 384 |
+
|
| 385 |
+
## 🔧 URL Rewriting Flow (Technical Details)
|
| 386 |
+
|
| 387 |
+
```
|
| 388 |
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
| 389 |
+
│ URL REWRITING LOGIC │
|
| 390 |
+
└─────────────────────────────────────────────────────────────────────────────┘
|
| 391 |
+
|
| 392 |
+
Problem: HLS playlists contain relative file paths
|
| 393 |
+
Wasabi bucket is NOT public (free trial restriction)
|
| 394 |
+
Direct Wasabi URLs won't work
|
| 395 |
+
|
| 396 |
+
Solution: Rewrite ALL URLs to point to our proxy server
|
| 397 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 398 |
+
|
| 399 |
+
Original master.m3u8 from Wasabi:
|
| 400 |
+
─────────────────────────────────────────────────────────────────────────────
|
| 401 |
+
#EXTM3U
|
| 402 |
+
#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="Audio",DEFAULT=YES,URI="index_audio.m3u8"
|
| 403 |
+
#EXT-X-STREAM-INF:BANDWIDTH=1328000,RESOLUTION=854x480,CODECS="avc1.64001f,mp4a.40.2",AUDIO="audio"
|
| 404 |
+
index_480p.m3u8
|
| 405 |
+
#EXT-X-STREAM-INF:BANDWIDTH=3128000,RESOLUTION=1280x720,CODECS="avc1.64001f,mp4a.40.2",AUDIO="audio"
|
| 406 |
+
index_720p.m3u8
|
| 407 |
+
#EXT-X-STREAM-INF:BANDWIDTH=5628000,RESOLUTION=1920x1080,CODECS="avc1.64001f,mp4a.40.2",AUDIO="audio"
|
| 408 |
+
index_1080p.m3u8
|
| 409 |
+
─────────────────────────────────────────────────────────────────────────────
|
| 410 |
+
|
| 411 |
+
Rewriting Process (2 patterns):
|
| 412 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 413 |
+
|
| 414 |
+
Pattern 1: URIs inside quotes (audio track)
|
| 415 |
+
────────────────────────────────────────────────────────────────────────────
|
| 416 |
+
Regex: /URI="([^"]+\.(m3u8|ts))"/g
|
| 417 |
+
|
| 418 |
+
Before: URI="index_audio.m3u8"
|
| 419 |
+
After: URI="https://domain/wasabi/file?path=hls%2F{uploadId}%2Findex_audio.m3u8"
|
| 420 |
+
|
| 421 |
+
Pattern 2: Standalone filenames on their own line
|
| 422 |
+
────────────────────────────────────────────────────────────────────────────
|
| 423 |
+
Check: line.trim().endsWith('.m3u8') || line.trim().endsWith('.ts')
|
| 424 |
+
|
| 425 |
+
Before: index_480p.m3u8
|
| 426 |
+
After: https://domain/wasabi/file?path=hls%2F{uploadId}%2Findex_480p.m3u8
|
| 427 |
+
|
| 428 |
+
Before: segment_720p_000.ts
|
| 429 |
+
After: https://domain/wasabi/file?path=hls%2F{uploadId}%2Fsegment_720p_000.ts
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
Rewritten master.m3u8 sent to client:
|
| 433 |
+
─────────────────────────────────────────────────────────────────────────────
|
| 434 |
+
#EXTM3U
|
| 435 |
+
#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="Audio",DEFAULT=YES,URI="https://domain/wasabi/file?path=hls%2F1766933325669-42b7k6%2Findex_audio.m3u8"
|
| 436 |
+
#EXT-X-STREAM-INF:BANDWIDTH=1328000,RESOLUTION=854x480,CODECS="avc1.64001f,mp4a.40.2",AUDIO="audio"
|
| 437 |
+
https://domain/wasabi/file?path=hls%2F1766933325669-42b7k6%2Findex_480p.m3u8
|
| 438 |
+
#EXT-X-STREAM-INF:BANDWIDTH=3128000,RESOLUTION=1280x720,CODECS="avc1.64001f,mp4a.40.2",AUDIO="audio"
|
| 439 |
+
https://domain/wasabi/file?path=hls%2F1766933325669-42b7k6%2Findex_720p.m3u8
|
| 440 |
+
#EXT-X-STREAM-INF:BANDWIDTH=5628000,RESOLUTION=1920x1080,CODECS="avc1.64001f,mp4a.40.2",AUDIO="audio"
|
| 441 |
+
https://domain/wasabi/file?path=hls%2F1766933325669-42b7k6%2Findex_1080p.m3u8
|
| 442 |
+
─────────────────────────────────────────────────────────────────────────────
|
| 443 |
+
|
| 444 |
+
Result: HLS player sees fully qualified URLs
|
| 445 |
+
All URLs point to our server (proxy CDN)
|
| 446 |
+
Player fetches everything through our API
|
| 447 |
+
```
|
| 448 |
+
|
| 449 |
+
---
|
| 450 |
+
|
| 451 |
+
## 🐌 Why Local Testing is Slow
|
| 452 |
+
|
| 453 |
+
```
|
| 454 |
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
| 455 |
+
│ LOCAL TESTING BOTTLENECK ANALYSIS │
|
| 456 |
+
└─────────────────────────────────────────────────────────────────────────────┘
|
| 457 |
+
|
| 458 |
+
Scenario: Testing on localhost with slow home internet
|
| 459 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 460 |
+
|
| 461 |
+
Your Network:
|
| 462 |
+
┌──────────────────────────────────────────────────────────────────────────┐
|
| 463 |
+
│ │
|
| 464 |
+
│ [Your Computer] <───────────┐ │
|
| 465 |
+
│ ┌──────────────┐ │ │
|
| 466 |
+
│ │ Browser │ │ Loopback (instant) │
|
| 467 |
+
│ │ HLS Player │ │ │
|
| 468 |
+
│ └───────┬──────┘ │ │
|
| 469 |
+
│ │ │ │
|
| 470 |
+
│ │ HTTP Request │ │
|
| 471 |
+
│ │ (localhost:5119) │ │
|
| 472 |
+
│ ▼ │ │
|
| 473 |
+
│ ┌──────────────┐ │ │
|
| 474 |
+
│ │ NestJS Server│◄────────────┘ │
|
| 475 |
+
│ │ (localhost) │ │
|
| 476 |
+
│ └───────┬──────┘ │
|
| 477 |
+
│ │ │
|
| 478 |
+
│ │ ⚠️ BOTTLENECK: Downloads from Wasabi │
|
| 479 |
+
│ │ via your home internet (slow upload!) │
|
| 480 |
+
│ │ │
|
| 481 |
+
│ ▼ │
|
| 482 |
+
│ ┌─────────────────────┐ │
|
| 483 |
+
│ │ Home Internet │ Upload: 5-20 Mbps (typical home internet) │
|
| 484 |
+
│ │ Router/Modem │ Download: 50-200 Mbps │
|
| 485 |
+
│ └──────────┬──────────┘ │
|
| 486 |
+
└─────────────┼──────────────────────────────────────────────────────────┘
|
| 487 |
+
│
|
| 488 |
+
│ Internet (slow home upload speed)
|
| 489 |
+
▼
|
| 490 |
+
┌───────────────────┐
|
| 491 |
+
│ Wasabi S3 │ Located in: us-west-1 (California)
|
| 492 |
+
│ Storage │ Upload/Download: Very fast (datacenter speeds)
|
| 493 |
+
└───────────────────┘
|
| 494 |
+
|
| 495 |
+
|
| 496 |
+
The Problem (Request Flow):
|
| 497 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 498 |
+
|
| 499 |
+
1. Browser requests segment:
|
| 500 |
+
GET /wasabi/file?path=segment_720p_000.ts
|
| 501 |
+
|
| 502 |
+
2. Server receives request (instant - localhost)
|
| 503 |
+
|
| 504 |
+
3. ⚠️ Server downloads from Wasabi:
|
| 505 |
+
┌─────────────────────────────────────────────────────────────┐
|
| 506 |
+
│ NestJS Server │
|
| 507 |
+
│ └─> Wasabi S3 Request │
|
| 508 |
+
│ │
|
| 509 |
+
│ Download speed LIMITED BY: │
|
| 510 |
+
│ • Your home internet UPLOAD speed (5-20 Mbps typical) │
|
| 511 |
+
│ • Wasabi is downloading TO your home (uses upload speed!) │
|
| 512 |
+
│ • Segment size: ~1-2 MB │
|
| 513 |
+
│ │
|
| 514 |
+
│ Time: 1-2 MB ÷ 10 Mbps upload = 0.8-1.6 seconds per chunk │
|
| 515 |
+
└─────────────────────────────────────────────────────────────┘
|
| 516 |
+
|
| 517 |
+
4. Server streams to browser (instant - localhost)
|
| 518 |
+
|
| 519 |
+
5. ⚠️ Player needs 3-5 segments buffered before playing
|
| 520 |
+
Total time: 3 segments × 1.5s each = 4.5 seconds initial load
|
| 521 |
+
|
| 522 |
+
|
| 523 |
+
Why it's slow:
|
| 524 |
+
─────────────────────────────────────────────────────────────────────────────
|
| 525 |
+
❌ Home internet upload speed is slow (5-20 Mbps typical)
|
| 526 |
+
❌ Wasabi → Your Server uses your UPLOAD bandwidth
|
| 527 |
+
❌ Multiple segments needed before playback starts
|
| 528 |
+
❌ Each segment takes 1-2 seconds to download from Wasabi
|
| 529 |
+
|
| 530 |
+
Visual Timeline:
|
| 531 |
+
─────────────────────────────────────────────────────────────────────────────
|
| 532 |
+
Time Browser Server Wasabi
|
| 533 |
+
────────────────────────────────────────────────────���───────────────────────
|
| 534 |
+
0.0s Request seg_000.ts ──> Waiting
|
| 535 |
+
0.1s Request to Wasabi ──────> Processing
|
| 536 |
+
0.2s Downloading.............. Sending
|
| 537 |
+
0.5s Downloading.............. Sending
|
| 538 |
+
1.0s Downloading.............. Sending
|
| 539 |
+
1.5s Download complete <────── Done
|
| 540 |
+
1.6s Receiving chunk <──── Streaming to browser
|
| 541 |
+
2.0s Request seg_001.ts ──> Waiting (repeat above)
|
| 542 |
+
```
|
| 543 |
+
|
| 544 |
+
---
|
| 545 |
+
|
| 546 |
+
## 🚀 Production Performance (Why Server Deployment is Fast)
|
| 547 |
+
|
| 548 |
+
```
|
| 549 |
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
| 550 |
+
│ PRODUCTION DEPLOYMENT PERFORMANCE │
|
| 551 |
+
└─────────────────────────────────────────────────────────────────────────────┘
|
| 552 |
+
|
| 553 |
+
Scenario: Server deployed on cloud (AWS, DigitalOcean, Heroku, etc.)
|
| 554 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 555 |
+
|
| 556 |
+
Production Network:
|
| 557 |
+
┌──────────────────────────────────────────────────────────────────────────┐
|
| 558 |
+
│ CLIENT SIDE │
|
| 559 |
+
│ [User's Device] │
|
| 560 |
+
│ ┌──────────────┐ │
|
| 561 |
+
│ │ Browser │ │
|
| 562 |
+
│ │ HLS Player │ │
|
| 563 |
+
│ └───────┬──────┘ │
|
| 564 |
+
└──────────┼──────────────────────────────────────────────────────────────┘
|
| 565 |
+
│
|
| 566 |
+
│ Internet (user's speed doesn't affect server-to-wasabi)
|
| 567 |
+
│
|
| 568 |
+
▼
|
| 569 |
+
┌──────────────────────────────────────────────────────────────────────────┐
|
| 570 |
+
│ SERVER SIDE (CLOUD) │
|
| 571 |
+
│ ┌──────────────────────────────────────────────────────────────┐ │
|
| 572 |
+
│ │ Cloud Datacenter (AWS/DigitalOcean/Heroku) │ │
|
| 573 |
+
│ │ │ │
|
| 574 |
+
│ │ ┌──────────────┐ │ │
|
| 575 |
+
│ │ │ NestJS Server│ │ │
|
| 576 |
+
│ │ │ (Production) │ │ │
|
| 577 |
+
│ │ └───────┬──────┘ │ │
|
| 578 |
+
│ │ │ │ │
|
| 579 |
+
│ │ │ ⚡ FAST: Datacenter-to-datacenter connection │ │
|
| 580 |
+
│ │ │ Speed: 1-10 Gbps (1000-10000 Mbps) │ │
|
| 581 |
+
│ │ │ Latency: 5-20ms │ │
|
| 582 |
+
│ │ │ │ │
|
| 583 |
+
│ │ ▼ │ │
|
| 584 |
+
│ │ ┌───────────────────┐ │ │
|
| 585 |
+
│ │ │ High-Speed │ Upload: 1-10 Gbps │ │
|
| 586 |
+
│ │ │ Datacenter │ Download: 1-10 Gbps │ │
|
| 587 |
+
│ │ │ Network │ Unlimited bandwidth │ │
|
| 588 |
+
│ │ └────────┬──────────┘ │ │
|
| 589 |
+
│ └───────────┼────────────────────────────────────────────────┘ │
|
| 590 |
+
└──────────────┼���─────────────────────────────────────────────────────────┘
|
| 591 |
+
│
|
| 592 |
+
│ High-speed datacenter backbone
|
| 593 |
+
│ Speed: 1-10 Gbps
|
| 594 |
+
│ Latency: ~10ms
|
| 595 |
+
│
|
| 596 |
+
▼
|
| 597 |
+
┌────────────────────┐
|
| 598 |
+
│ Wasabi S3 │ Located in: us-west-1 (California)
|
| 599 |
+
│ Storage │ Datacenter: High-speed connections
|
| 600 |
+
└────────────────────┘
|
| 601 |
+
|
| 602 |
+
|
| 603 |
+
Performance Comparison:
|
| 604 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 605 |
+
|
| 606 |
+
Local Testing Production Server
|
| 607 |
+
────────────────────────────────────────────────────────────────────────────
|
| 608 |
+
Server Network Home Internet Cloud Datacenter
|
| 609 |
+
Upload Speed 5-20 Mbps 1,000-10,000 Mbps
|
| 610 |
+
Download Speed 50-200 Mbps 1,000-10,000 Mbps
|
| 611 |
+
Server→Wasabi Uses upload (slow) Datacenter speed (fast)
|
| 612 |
+
Latency 50-100ms 5-20ms
|
| 613 |
+
Segment Download 1-2 seconds 0.05-0.2 seconds
|
| 614 |
+
Initial Load 4-6 seconds 0.5-1 second
|
| 615 |
+
Buffering Frequent Rare/Never
|
| 616 |
+
|
| 617 |
+
|
| 618 |
+
Production Request Flow (FAST!):
|
| 619 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 620 |
+
|
| 621 |
+
1. Browser requests segment:
|
| 622 |
+
GET /wasabi/file?path=segment_720p_000.ts
|
| 623 |
+
|
| 624 |
+
2. Server receives request (fast - CDN)
|
| 625 |
+
|
| 626 |
+
3. ✅ Server downloads from Wasabi:
|
| 627 |
+
┌─────────────────────────────────────────────────────────────┐
|
| 628 |
+
│ NestJS Server (Cloud Datacenter) │
|
| 629 |
+
│ └─> Wasabi S3 Request │
|
| 630 |
+
│ │
|
| 631 |
+
│ Download speed: │
|
| 632 |
+
│ • Datacenter connection: 1-10 Gbps │
|
| 633 |
+
│ • Wasabi datacenter: 1-10 Gbps │
|
| 634 |
+
│ • Segment size: ~1-2 MB │
|
| 635 |
+
│ │
|
| 636 |
+
│ Time: 1-2 MB ÷ 1 Gbps = 0.01-0.02 seconds per chunk │
|
| 637 |
+
│ (50-100x FASTER than home internet!) │
|
| 638 |
+
└─────────────────────────────────────────────────────────────┘
|
| 639 |
+
|
| 640 |
+
4. Server streams to browser (depends on user's internet)
|
| 641 |
+
|
| 642 |
+
5. ✅ Player starts playing almost immediately
|
| 643 |
+
Total time: 3 segments × 0.05s each = 0.15 seconds initial load
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
Visual Timeline (Production):
|
| 647 |
+
─────────────────────────────────────────────────────────────────────────────
|
| 648 |
+
Time Browser Server (Cloud) Wasabi
|
| 649 |
+
────────────────────────────────────────────────────────────────────────────
|
| 650 |
+
0.00s Request seg_000.ts ──> Waiting
|
| 651 |
+
0.01s Request to Wasabi ──────> Processing
|
| 652 |
+
0.02s Download complete <────── Done (FAST!)
|
| 653 |
+
0.03s Receiving chunk <──── Streaming to browser
|
| 654 |
+
0.10s Request seg_001.ts ──> Waiting
|
| 655 |
+
0.11s Request to Wasabi ──────> Processing
|
| 656 |
+
0.12s Download complete <────── Done (FAST!)
|
| 657 |
+
0.13s Receiving chunk <──── Streaming to browser
|
| 658 |
+
0.20s 🎬 Video starts playing (smooth!)
|
| 659 |
+
|
| 660 |
+
|
| 661 |
+
Why Production is Fast:
|
| 662 |
+
─────────────────────────────────────────────────────────────────────────────
|
| 663 |
+
✅ Datacenter network: 1-10 Gbps (100-1000x faster than home)
|
| 664 |
+
✅ Low latency: 5-20ms (vs 50-100ms home internet)
|
| 665 |
+
✅ Unlimited bandwidth: No ISP throttling
|
| 666 |
+
✅ Parallel downloads: Server can handle 100+ concurrent requests
|
| 667 |
+
✅ Geographic proximity: Server and Wasabi both in datacenters
|
| 668 |
+
|
| 669 |
+
|
| 670 |
+
Optimization Techniques Applied:
|
| 671 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 672 |
+
|
| 673 |
+
1. ⚡ Direct Piping (No Buffering)
|
| 674 |
+
─────────────────────────────────────────────────────────────────────────
|
| 675 |
+
result.stream.pipe(res)
|
| 676 |
+
|
| 677 |
+
• Chunks stream immediately (no waiting for full download)
|
| 678 |
+
• Server memory stays low (no buffering in RAM)
|
| 679 |
+
• Start streaming to client while still downloading from Wasabi
|
| 680 |
+
|
| 681 |
+
2. ⚡ No Processing Overhead
|
| 682 |
+
─────────────────────────────────────────────────────────────────────────
|
| 683 |
+
For .ts segments: Pure streaming (no URL rewriting)
|
| 684 |
+
For .m3u8 files: Only rewrite text (very fast string operation)
|
| 685 |
+
|
| 686 |
+
3. ⚡ HTTP Range Support
|
| 687 |
+
─────────────────────────────────────────────────────────────────────────
|
| 688 |
+
Accept-Ranges: bytes
|
| 689 |
+
|
| 690 |
+
• Allows video seeking (jump to any point)
|
| 691 |
+
• Browser only downloads needed chunks
|
| 692 |
+
• No need to download entire video
|
| 693 |
+
|
| 694 |
+
4. ⚡ CORS Enabled
|
| 695 |
+
─────────────────────────────────────────────────────────────────────────
|
| 696 |
+
Access-Control-Allow-Origin: *
|
| 697 |
+
|
| 698 |
+
• Works with any HLS player
|
| 699 |
+
• No preflight request delays
|
| 700 |
+
|
| 701 |
+
5. ⚡ CDN-Ready Headers
|
| 702 |
+
─────────────────────────────────────────────────────────────────────────
|
| 703 |
+
Cache-Control: public, max-age=2592000
|
| 704 |
+
|
| 705 |
+
• Browser caches segments for 30 days
|
| 706 |
+
• Repeated views = instant playback
|
| 707 |
+
• Can add Cloudflare/CDN in front for global edge caching
|
| 708 |
+
```
|
| 709 |
+
|
| 710 |
+
---
|
| 711 |
+
|
| 712 |
+
## 📊 Performance Comparison Summary
|
| 713 |
+
|
| 714 |
+
```
|
| 715 |
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
| 716 |
+
│ PERFORMANCE METRICS COMPARISON │
|
| 717 |
+
└─────────────────────────────────────────────────────────────────────────────┘
|
| 718 |
+
|
| 719 |
+
Metric Local Testing Production Server Improvement
|
| 720 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 721 |
+
Server Network Speed 10 Mbps 1,000 Mbps 100x faster
|
| 722 |
+
Server→Wasabi Latency 50-100ms 5-20ms 5x faster
|
| 723 |
+
Segment Download Time 1-2 seconds 0.05-0.2 seconds 10-20x faster
|
| 724 |
+
Initial Video Load 4-6 seconds 0.5-1 second 5-10x faster
|
| 725 |
+
Buffering Frequency Every 3-5 seg Rare/Never Smooth
|
| 726 |
+
Concurrent Users 1-2 100-1000+ Scalable
|
| 727 |
+
Memory Usage Low (piping) Low (piping) Efficient
|
| 728 |
+
CPU Usage Low (no proc) Low (no proc) Efficient
|
| 729 |
+
|
| 730 |
+
|
| 731 |
+
Bottleneck Analysis:
|
| 732 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 733 |
+
|
| 734 |
+
Local Testing:
|
| 735 |
+
┌────────────────────────────────────────────────────────────────────────────┐
|
| 736 |
+
│ │
|
| 737 |
+
│ Browser ←─── [INSTANT] ───→ Server ←─── [SLOW!] ───→ Wasabi │
|
| 738 |
+
│ (Fast) (Localhost) (Home Internet) │
|
| 739 |
+
│ ↑ │
|
| 740 |
+
│ └── BOTTLENECK HERE │
|
| 741 |
+
│ (5-20 Mbps upload) │
|
| 742 |
+
└────────────────────────────────────────────────────────────────────────────┘
|
| 743 |
+
|
| 744 |
+
Production:
|
| 745 |
+
┌────────────────────────────────────────────────────────────────────────────┐
|
| 746 |
+
│ │
|
| 747 |
+
│ Browser ←─ [User Speed] ─→ Server ←─── [FAST!] ───→ Wasabi │
|
| 748 |
+
│ (Varies) (Cloud) (Datacenter) │
|
| 749 |
+
│ ↑ │
|
| 750 |
+
│ └── NO BOTTLENECK │
|
| 751 |
+
│ (1-10 Gbps) │
|
| 752 |
+
└────────────────────────────────────────────────────────────────────────────┘
|
| 753 |
+
|
| 754 |
+
|
| 755 |
+
Key Takeaway:
|
| 756 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 757 |
+
📌 Local testing is slow because your HOME INTERNET UPLOAD SPEED limits
|
| 758 |
+
the server's ability to download from Wasabi.
|
| 759 |
+
|
| 760 |
+
📌 Production is fast because SERVER-TO-WASABI happens at datacenter speeds
|
| 761 |
+
(1000x faster than home internet).
|
| 762 |
+
|
| 763 |
+
📌 Your slow internet does NOT affect the production server's performance!
|
| 764 |
+
|
| 765 |
+
📌 Deploy to cloud → Experience true speed! 🚀
|
| 766 |
+
```
|
| 767 |
+
|
| 768 |
+
---
|
| 769 |
+
|
| 770 |
+
## 🎯 Architecture Benefits
|
| 771 |
+
|
| 772 |
+
```
|
| 773 |
+
┌─────────────────────────────────────────────────────────────────────────────┐
|
| 774 |
+
│ WHY THIS ARCHITECTURE? │
|
| 775 |
+
└─────────────────────────────────────────────────────────────────────────────┘
|
| 776 |
+
|
| 777 |
+
✅ Works with Wasabi Free Trial
|
| 778 |
+
• No need for public bucket access
|
| 779 |
+
• No need to pay for CDN features
|
| 780 |
+
• Full control over access
|
| 781 |
+
|
| 782 |
+
✅ Security
|
| 783 |
+
• All Wasabi credentials stay on server
|
| 784 |
+
• No direct Wasabi URLs exposed to clients
|
| 785 |
+
• Can add authentication/authorization later
|
| 786 |
+
|
| 787 |
+
✅ Flexibility
|
| 788 |
+
• Can add rate limiting per user
|
| 789 |
+
• Can track analytics (views, bandwidth)
|
| 790 |
+
• Can add watermarks or DRM later
|
| 791 |
+
|
| 792 |
+
✅ Cost Effective
|
| 793 |
+
• Wasabi storage: Very cheap ($6/TB/month)
|
| 794 |
+
• No egress fees (unlimited downloads)
|
| 795 |
+
• No need for separate CDN service
|
| 796 |
+
|
| 797 |
+
✅ Scalability
|
| 798 |
+
• Direct piping = Low memory usage
|
| 799 |
+
• Can handle 100+ concurrent streams
|
| 800 |
+
• Can add Cloudflare/CDN layer later for global reach
|
| 801 |
+
|
| 802 |
+
✅ Performance
|
| 803 |
+
• Direct piping = Fast streaming
|
| 804 |
+
• No buffering on server = Low latency
|
| 805 |
+
• Works with any HLS player
|
| 806 |
+
|
| 807 |
+
⚠️ Trade-off: Server bandwidth
|
| 808 |
+
• All video traffic goes through your server
|
| 809 |
+
• For high traffic, add CDN layer (Cloudflare) in front
|
| 810 |
+
• But for most apps, this is fine!
|
| 811 |
+
```
|
| 812 |
+
|
| 813 |
+
---
|
| 814 |
+
|
| 815 |
+
## 🔍 Debugging Tips
|
| 816 |
+
|
| 817 |
+
```
|
| 818 |
+
When video is slow/not loading:
|
| 819 |
+
═══════════════════════════════════════════════════════════════════════════════
|
| 820 |
+
|
| 821 |
+
1. Check server logs:
|
| 822 |
+
• Look for "Streaming file:" messages
|
| 823 |
+
• Check for errors in WasabiStorageService
|
| 824 |
+
• Verify uploadId exists in Wasabi
|
| 825 |
+
|
| 826 |
+
2. Check network tab in browser:
|
| 827 |
+
• Are requests returning 200 OK?
|
| 828 |
+
• Check Content-Type headers (should be video/mp2t for segments)
|
| 829 |
+
• Look at response times (should be fast in production)
|
| 830 |
+
|
| 831 |
+
3. Check Wasabi bucket:
|
| 832 |
+
• Verify files exist: hls/{uploadId}/master.m3u8
|
| 833 |
+
• Check file sizes (master.m3u8 should be small, segments ~1-2 MB)
|
| 834 |
+
|
| 835 |
+
4. Test with curl:
|
| 836 |
+
curl "http://localhost:5119/wasabi/hls?uploadId=1766933325669-42b7k6"
|
| 837 |
+
# Should return rewritten m3u8 content
|
| 838 |
+
|
| 839 |
+
5. Check APP_URL in .env:
|
| 840 |
+
• Must match your actual domain
|
| 841 |
+
• Local: http://localhost:5119
|
| 842 |
+
• Prod: https://yourdomain.com
|
| 843 |
+
```
|
| 844 |
+
|
| 845 |
+
---
|
| 846 |
+
|
| 847 |
+
## 📝 Summary
|
| 848 |
+
|
| 849 |
+
**Upload Flow:** Video → FFmpeg HLS Conversion → Upload to Wasabi → Cleanup Local Files
|
| 850 |
+
|
| 851 |
+
**Streaming Flow:** Browser → Server (proxy) → Wasabi → Server → Browser
|
| 852 |
+
|
| 853 |
+
**URL Rewriting:** All playlists rewritten to point to server proxy endpoints
|
| 854 |
+
|
| 855 |
+
**Performance:**
|
| 856 |
+
|
| 857 |
+
- Local testing: Slow (limited by home internet upload speed)
|
| 858 |
+
- Production: Fast (datacenter-to-datacenter speeds, 100x faster)
|
| 859 |
+
|
| 860 |
+
**Key Insight:** Your slow local internet does NOT affect production performance! 🚀
|
|
@@ -0,0 +1,513 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 🎬 Wasabi HLS Streaming - Complete Mermaid Diagram
|
| 2 |
+
|
| 3 |
+
This is a comprehensive visual diagram showing the entire HLS video streaming architecture from upload to playback.
|
| 4 |
+
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
## 📊 Complete Architecture Flow Diagram
|
| 8 |
+
|
| 9 |
+
```mermaid
|
| 10 |
+
flowchart TB
|
| 11 |
+
subgraph ClientUpload["🎥 CLIENT - VIDEO UPLOAD"]
|
| 12 |
+
A["User Selects Video File
|
| 13 |
+
video.mp4 - 500 MB"]
|
| 14 |
+
B["Browser Uploads via FormData
|
| 15 |
+
POST /wasabi/upload-video-hls"]
|
| 16 |
+
end
|
| 17 |
+
|
| 18 |
+
subgraph ControllerUpload["🎛️ CONTROLLER - Upload Endpoint"]
|
| 19 |
+
C["WasabiStorageController
|
| 20 |
+
@Post upload-video-hls
|
| 21 |
+
@UseInterceptors FileInterceptor"]
|
| 22 |
+
D["Multer saves to disk
|
| 23 |
+
./resources/temp/
|
| 24 |
+
video-1766933325669.mp4"]
|
| 25 |
+
E["Generate Upload ID
|
| 26 |
+
1766933325669-42b7k6"]
|
| 27 |
+
end
|
| 28 |
+
|
| 29 |
+
subgraph ServiceUpload["⚙️ SERVICE - Start Processing"]
|
| 30 |
+
F["WasabiStorageService
|
| 31 |
+
startVideoProcessing"]
|
| 32 |
+
G["Return Upload ID immediately
|
| 33 |
+
Non-blocking async processing"]
|
| 34 |
+
H["Start processVideoAsync
|
| 35 |
+
in background"]
|
| 36 |
+
end
|
| 37 |
+
|
| 38 |
+
subgraph HLSConversion["🎬 HLS CONVERSION - FFmpeg Processing"]
|
| 39 |
+
I["HlsConversionService
|
| 40 |
+
convertVideoToAdaptiveHLS"]
|
| 41 |
+
J["FFmpeg Converts Video
|
| 42 |
+
Creates Multiple Qualities"]
|
| 43 |
+
|
| 44 |
+
subgraph Qualities["Video Qualities Generated"]
|
| 45 |
+
K1["1080p: 1920x1080
|
| 46 |
+
Bitrate: 5500k
|
| 47 |
+
Preset: veryfast"]
|
| 48 |
+
K2["720p: 1280x720
|
| 49 |
+
Bitrate: 3000k
|
| 50 |
+
Preset: veryfast"]
|
| 51 |
+
K3["480p: 854x480
|
| 52 |
+
Bitrate: 1200k
|
| 53 |
+
Preset: veryfast"]
|
| 54 |
+
K4["Audio Only
|
| 55 |
+
Bitrate: 128k
|
| 56 |
+
Format: AAC"]
|
| 57 |
+
end
|
| 58 |
+
|
| 59 |
+
L["Generate M3U8 Playlists
|
| 60 |
+
Master + Quality Playlists"]
|
| 61 |
+
M["Generate TS Segments
|
| 62 |
+
6 seconds each chunk"]
|
| 63 |
+
|
| 64 |
+
subgraph FilesGenerated["📁 Files Created in /tmp/streamflix-hls/uploadId/"]
|
| 65 |
+
N1["master.m3u8
|
| 66 |
+
Main playlist with all qualities"]
|
| 67 |
+
N2["index_1080p.m3u8
|
| 68 |
+
1080p playlist"]
|
| 69 |
+
N3["index_720p.m3u8
|
| 70 |
+
720p playlist"]
|
| 71 |
+
N4["index_480p.m3u8
|
| 72 |
+
480p playlist"]
|
| 73 |
+
N5["index_audio.m3u8
|
| 74 |
+
Audio-only playlist"]
|
| 75 |
+
N6["segment_1080p_000.ts to XXX.ts
|
| 76 |
+
1080p video chunks"]
|
| 77 |
+
N7["segment_720p_000.ts to XXX.ts
|
| 78 |
+
720p video chunks"]
|
| 79 |
+
N8["segment_480p_000.ts to XXX.ts
|
| 80 |
+
480p video chunks"]
|
| 81 |
+
N9["segment_audio_000.ts to XXX.ts
|
| 82 |
+
Audio chunks"]
|
| 83 |
+
end
|
| 84 |
+
end
|
| 85 |
+
|
| 86 |
+
subgraph WasabiUpload["☁️ WASABI UPLOAD - S3 Storage"]
|
| 87 |
+
O["WasabiStorageService
|
| 88 |
+
uploadHLSFilesToWasabi"]
|
| 89 |
+
P["Upload master.m3u8
|
| 90 |
+
to hls/uploadId/master.m3u8"]
|
| 91 |
+
Q["Upload all quality playlists
|
| 92 |
+
index_*.m3u8"]
|
| 93 |
+
R["Upload all TS segments
|
| 94 |
+
segment_*.ts"]
|
| 95 |
+
|
| 96 |
+
subgraph WasabiStorage["🗄️ Wasabi S3 Bucket: streamflix-backed"]
|
| 97 |
+
S1["hls/
|
| 98 |
+
1766933325669-42b7k6/
|
| 99 |
+
master.m3u8"]
|
| 100 |
+
S2["hls/
|
| 101 |
+
1766933325669-42b7k6/
|
| 102 |
+
index_1080p.m3u8"]
|
| 103 |
+
S3["hls/
|
| 104 |
+
1766933325669-42b7k6/
|
| 105 |
+
index_720p.m3u8"]
|
| 106 |
+
S4["hls/
|
| 107 |
+
1766933325669-42b7k6/
|
| 108 |
+
index_480p.m3u8"]
|
| 109 |
+
S5["hls/
|
| 110 |
+
1766933325669-42b7k6/
|
| 111 |
+
index_audio.m3u8"]
|
| 112 |
+
S6["hls/
|
| 113 |
+
1766933325669-42b7k6/
|
| 114 |
+
segment_*.ts
|
| 115 |
+
100+ files"]
|
| 116 |
+
end
|
| 117 |
+
|
| 118 |
+
T["Cleanup Local Files
|
| 119 |
+
Delete /tmp/streamflix-hls/uploadId/
|
| 120 |
+
Delete ./resources/temp/video-*.mp4"]
|
| 121 |
+
end
|
| 122 |
+
|
| 123 |
+
subgraph ClientPlayback["📱 CLIENT - VIDEO PLAYBACK"]
|
| 124 |
+
U["User Opens HLS Player
|
| 125 |
+
Pastes M3U8 URL"]
|
| 126 |
+
V["GET /wasabi/hls?uploadId=1766933325669-42b7k6"]
|
| 127 |
+
end
|
| 128 |
+
|
| 129 |
+
subgraph ControllerStream["🎛️ CONTROLLER - HLS Endpoint"]
|
| 130 |
+
W["WasabiStorageController
|
| 131 |
+
@Get hls
|
| 132 |
+
@SkipApiToken Public"]
|
| 133 |
+
X["Call getRewrittenM3U8 service"]
|
| 134 |
+
end
|
| 135 |
+
|
| 136 |
+
subgraph ServiceMaster["⚙️ SERVICE - Master Playlist"]
|
| 137 |
+
Y["WasabiStorageService
|
| 138 |
+
getRewrittenM3U8"]
|
| 139 |
+
Z["Fetch master.m3u8 from Wasabi
|
| 140 |
+
Internal S3 GetObjectCommand"]
|
| 141 |
+
|
| 142 |
+
subgraph WasabiFetch1["☁️ Wasabi Returns Original"]
|
| 143 |
+
AA["master.m3u8 content:
|
| 144 |
+
#EXTM3U
|
| 145 |
+
#EXT-X-MEDIA:URI=index_audio.m3u8
|
| 146 |
+
index_480p.m3u8
|
| 147 |
+
index_720p.m3u8
|
| 148 |
+
index_1080p.m3u8"]
|
| 149 |
+
end
|
| 150 |
+
|
| 151 |
+
AB["URL Rewriting Process
|
| 152 |
+
Convert relative paths to absolute URLs"]
|
| 153 |
+
|
| 154 |
+
subgraph URLRewrite1["🔄 URL Rewriting Logic"]
|
| 155 |
+
AC["Pattern 1: URI in quotes
|
| 156 |
+
URI=index_audio.m3u8
|
| 157 |
+
↓
|
| 158 |
+
URI=https://domain/wasabi/file?path=hls%2F...%2Findex_audio.m3u8"]
|
| 159 |
+
AD["Pattern 2: Standalone files
|
| 160 |
+
index_720p.m3u8
|
| 161 |
+
↓
|
| 162 |
+
https://domain/wasabi/file?path=hls%2F...%2Findex_720p.m3u8"]
|
| 163 |
+
end
|
| 164 |
+
|
| 165 |
+
AE["Return Rewritten Master Playlist
|
| 166 |
+
All URLs point to server proxy"]
|
| 167 |
+
end
|
| 168 |
+
|
| 169 |
+
subgraph ClientReceives1["📱 CLIENT - Receives Master"]
|
| 170 |
+
AF["HLS Player receives master.m3u8
|
| 171 |
+
Sees list of quality options"]
|
| 172 |
+
AG["Player selects quality
|
| 173 |
+
e.g., 720p"]
|
| 174 |
+
AH["GET /wasabi/file?path=hls/.../index_720p.m3u8"]
|
| 175 |
+
end
|
| 176 |
+
|
| 177 |
+
subgraph ControllerFile["🎛️ CONTROLLER - File Proxy"]
|
| 178 |
+
AI["WasabiStorageController
|
| 179 |
+
@Get file
|
| 180 |
+
@SkipApiToken Public"]
|
| 181 |
+
AJ["Call streamFile service
|
| 182 |
+
path: hls/uploadId/index_720p.m3u8"]
|
| 183 |
+
end
|
| 184 |
+
|
| 185 |
+
subgraph ServiceQuality["⚙️ SERVICE - Quality Playlist"]
|
| 186 |
+
AK["WasabiStorageService
|
| 187 |
+
streamFile"]
|
| 188 |
+
AL["Detect .m3u8 file in hls/ folder
|
| 189 |
+
Needs URL rewriting"]
|
| 190 |
+
AM["Fetch index_720p.m3u8 from Wasabi
|
| 191 |
+
GetObjectCommand"]
|
| 192 |
+
|
| 193 |
+
subgraph WasabiFetch2["☁️ Wasabi Returns Original"]
|
| 194 |
+
AN["index_720p.m3u8 content:
|
| 195 |
+
#EXTM3U
|
| 196 |
+
#EXTINF:6.000000,
|
| 197 |
+
segment_720p_000.ts
|
| 198 |
+
#EXTINF:6.000000,
|
| 199 |
+
segment_720p_001.ts
|
| 200 |
+
segment_720p_002.ts
|
| 201 |
+
..."]
|
| 202 |
+
end
|
| 203 |
+
|
| 204 |
+
AO["URL Rewriting Process
|
| 205 |
+
Convert segment paths"]
|
| 206 |
+
|
| 207 |
+
subgraph URLRewrite2["🔄 Segment URL Rewriting"]
|
| 208 |
+
AP["segment_720p_000.ts
|
| 209 |
+
↓
|
| 210 |
+
https://domain/wasabi/file?path=hls%2F...%2Fsegment_720p_000.ts"]
|
| 211 |
+
AQ["segment_720p_001.ts
|
| 212 |
+
↓
|
| 213 |
+
https://domain/wasabi/file?path=hls%2F...%2Fsegment_720p_001.ts"]
|
| 214 |
+
end
|
| 215 |
+
|
| 216 |
+
AR["Return Rewritten Quality Playlist
|
| 217 |
+
All segment URLs absolute"]
|
| 218 |
+
end
|
| 219 |
+
|
| 220 |
+
subgraph ClientReceives2["📱 CLIENT - Receives Quality Playlist"]
|
| 221 |
+
AS["HLS Player receives index_720p.m3u8
|
| 222 |
+
Sees list of video segments"]
|
| 223 |
+
AT["Player starts downloading segments
|
| 224 |
+
Parallel requests for buffering"]
|
| 225 |
+
AU1["GET /wasabi/file?path=.../segment_720p_000.ts"]
|
| 226 |
+
AU2["GET /wasabi/file?path=.../segment_720p_001.ts"]
|
| 227 |
+
AU3["GET /wasabi/file?path=.../segment_720p_002.ts"]
|
| 228 |
+
end
|
| 229 |
+
|
| 230 |
+
subgraph ControllerSegment["🎛️ CONTROLLER - Segment Proxy"]
|
| 231 |
+
AV["WasabiStorageController
|
| 232 |
+
@Get file
|
| 233 |
+
Multiple parallel requests"]
|
| 234 |
+
AW["Call streamFile service
|
| 235 |
+
path: hls/uploadId/segment_720p_000.ts"]
|
| 236 |
+
end
|
| 237 |
+
|
| 238 |
+
subgraph ServiceSegment["⚙️ SERVICE - Stream Video Chunk"]
|
| 239 |
+
AX["WasabiStorageService
|
| 240 |
+
streamFile"]
|
| 241 |
+
AY["Detect .ts file
|
| 242 |
+
No rewriting needed"]
|
| 243 |
+
AZ["Fetch segment from Wasabi
|
| 244 |
+
GetObjectCommand"]
|
| 245 |
+
|
| 246 |
+
subgraph WasabiFetch3["☁️ Wasabi Returns Segment"]
|
| 247 |
+
BA["segment_720p_000.ts
|
| 248 |
+
Binary video data
|
| 249 |
+
~1-2 MB size
|
| 250 |
+
6 seconds of video"]
|
| 251 |
+
end
|
| 252 |
+
|
| 253 |
+
BB["Direct Piping Stream
|
| 254 |
+
result.stream.pipe res
|
| 255 |
+
No buffering on server"]
|
| 256 |
+
end
|
| 257 |
+
|
| 258 |
+
subgraph ControllerResponse["🎛️ CONTROLLER - Stream Response"]
|
| 259 |
+
BC["Set Response Headers
|
| 260 |
+
Content-Type: video/mp2t
|
| 261 |
+
Accept-Ranges: bytes
|
| 262 |
+
Cache-Control: max-age=2592000
|
| 263 |
+
Access-Control-Allow-Origin: *"]
|
| 264 |
+
BD["Pipe stream to client
|
| 265 |
+
Chunks sent immediately"]
|
| 266 |
+
end
|
| 267 |
+
|
| 268 |
+
subgraph ClientPlayback2["📱 CLIENT - Video Playing"]
|
| 269 |
+
BE["Browser receives video chunks
|
| 270 |
+
Real-time streaming"]
|
| 271 |
+
BF["HLS Player buffers segments
|
| 272 |
+
3-5 segments ahead"]
|
| 273 |
+
BG["🎬 VIDEO PLAYING!
|
| 274 |
+
User watches content
|
| 275 |
+
Smooth playback"]
|
| 276 |
+
BH["Player continues downloading
|
| 277 |
+
next segments in background"]
|
| 278 |
+
BI["User can seek/jump
|
| 279 |
+
Player requests new segments
|
| 280 |
+
via Range requests"]
|
| 281 |
+
end
|
| 282 |
+
|
| 283 |
+
subgraph Performance["⚡ PERFORMANCE ANALYSIS"]
|
| 284 |
+
subgraph LocalTesting["🏠 LOCAL TESTING Slow"]
|
| 285 |
+
BJ["Your Computer
|
| 286 |
+
Home Internet"]
|
| 287 |
+
BK["Upload: 5-20 Mbps
|
| 288 |
+
Download: 50-200 Mbps"]
|
| 289 |
+
BL["Server → Wasabi
|
| 290 |
+
Uses YOUR upload speed
|
| 291 |
+
BOTTLENECK!"]
|
| 292 |
+
BM["Segment download: 1-2 sec
|
| 293 |
+
Initial load: 4-6 sec
|
| 294 |
+
Buffering: Frequent"]
|
| 295 |
+
end
|
| 296 |
+
|
| 297 |
+
subgraph ProductionServer["🚀 PRODUCTION FAST"]
|
| 298 |
+
BN["Cloud Datacenter
|
| 299 |
+
AWS/DigitalOcean/Heroku"]
|
| 300 |
+
BO["Upload: 1-10 Gbps
|
| 301 |
+
Download: 1-10 Gbps"]
|
| 302 |
+
BP["Server → Wasabi
|
| 303 |
+
Datacenter speeds
|
| 304 |
+
1000x FASTER!"]
|
| 305 |
+
BQ["Segment download: 0.05-0.2 sec
|
| 306 |
+
Initial load: 0.5-1 sec
|
| 307 |
+
Buffering: Rare/Never"]
|
| 308 |
+
end
|
| 309 |
+
end
|
| 310 |
+
|
| 311 |
+
%% Upload Flow Connections
|
| 312 |
+
A --> B
|
| 313 |
+
B --> C
|
| 314 |
+
C --> D
|
| 315 |
+
D --> E
|
| 316 |
+
E --> F
|
| 317 |
+
F --> G
|
| 318 |
+
F --> H
|
| 319 |
+
H --> I
|
| 320 |
+
I --> J
|
| 321 |
+
J --> K1 & K2 & K3 & K4
|
| 322 |
+
K1 & K2 & K3 & K4 --> L
|
| 323 |
+
L --> M
|
| 324 |
+
M --> N1 & N2 & N3 & N4 & N5 & N6 & N7 & N8 & N9
|
| 325 |
+
N1 & N2 & N3 & N4 & N5 & N6 & N7 & N8 & N9 --> O
|
| 326 |
+
O --> P & Q & R
|
| 327 |
+
P --> S1
|
| 328 |
+
Q --> S2 & S3 & S4 & S5
|
| 329 |
+
R --> S6
|
| 330 |
+
S1 & S2 & S3 & S4 & S5 & S6 --> T
|
| 331 |
+
|
| 332 |
+
%% Streaming Flow Connections
|
| 333 |
+
U --> V
|
| 334 |
+
V --> W
|
| 335 |
+
W --> X
|
| 336 |
+
X --> Y
|
| 337 |
+
Y --> Z
|
| 338 |
+
Z --> AA
|
| 339 |
+
AA --> AB
|
| 340 |
+
AB --> AC & AD
|
| 341 |
+
AC & AD --> AE
|
| 342 |
+
AE --> AF
|
| 343 |
+
AF --> AG
|
| 344 |
+
AG --> AH
|
| 345 |
+
AH --> AI
|
| 346 |
+
AI --> AJ
|
| 347 |
+
AJ --> AK
|
| 348 |
+
AK --> AL
|
| 349 |
+
AL --> AM
|
| 350 |
+
AM --> AN
|
| 351 |
+
AN --> AO
|
| 352 |
+
AO --> AP & AQ
|
| 353 |
+
AP & AQ --> AR
|
| 354 |
+
AR --> AS
|
| 355 |
+
AS --> AT
|
| 356 |
+
AT --> AU1 & AU2 & AU3
|
| 357 |
+
AU1 & AU2 & AU3 --> AV
|
| 358 |
+
AV --> AW
|
| 359 |
+
AW --> AX
|
| 360 |
+
AX --> AY
|
| 361 |
+
AY --> AZ
|
| 362 |
+
AZ --> BA
|
| 363 |
+
BA --> BB
|
| 364 |
+
BB --> BC
|
| 365 |
+
BC --> BD
|
| 366 |
+
BD --> BE
|
| 367 |
+
BE --> BF
|
| 368 |
+
BF --> BG
|
| 369 |
+
BG --> BH
|
| 370 |
+
BH --> BI
|
| 371 |
+
|
| 372 |
+
%% Performance Comparison
|
| 373 |
+
T -.-> BJ
|
| 374 |
+
BJ --> BK --> BL --> BM
|
| 375 |
+
T -.-> BN
|
| 376 |
+
BN --> BO --> BP --> BQ
|
| 377 |
+
|
| 378 |
+
%% Styling
|
| 379 |
+
classDef clientStyle fill:#4A90E2,stroke:#2E5C8A,stroke-width:3px,color:#fff
|
| 380 |
+
classDef controllerStyle fill:#E94B3C,stroke:#A63428,stroke-width:3px,color:#fff
|
| 381 |
+
classDef serviceStyle fill:#50C878,stroke:#2E7D4E,stroke-width:3px,color:#fff
|
| 382 |
+
classDef wasabiStyle fill:#FF6B35,stroke:#C44D26,stroke-width:3px,color:#fff
|
| 383 |
+
classDef hlsStyle fill:#9B59B6,stroke:#6C3D80,stroke-width:3px,color:#fff
|
| 384 |
+
classDef perfStyle fill:#F39C12,stroke:#C87F0A,stroke-width:3px,color:#fff
|
| 385 |
+
classDef rewriteStyle fill:#1ABC9C,stroke:#148F77,stroke-width:3px,color:#fff
|
| 386 |
+
|
| 387 |
+
class A,B,U,V,AF,AG,AH,AS,AT,AU1,AU2,AU3,BE,BF,BG,BH,BI clientStyle
|
| 388 |
+
class C,D,E,W,X,AI,AJ,AV,AW,BC,BD controllerStyle
|
| 389 |
+
class F,G,H,O,P,Q,R,T,Y,Z,AE,AK,AL,AM,AO,AR,AX,AY,AZ,BB serviceStyle
|
| 390 |
+
class S1,S2,S3,S4,S5,S6,AA,AN,BA wasabiStyle
|
| 391 |
+
class I,J,K1,K2,K3,K4,L,M,N1,N2,N3,N4,N5,N6,N7,N8,N9 hlsStyle
|
| 392 |
+
class AB,AC,AD,AP,AQ rewriteStyle
|
| 393 |
+
class BJ,BK,BL,BM,BN,BO,BP,BQ perfStyle
|
| 394 |
+
```
|
| 395 |
+
|
| 396 |
+
---
|
| 397 |
+
|
| 398 |
+
## 🎯 Diagram Legend
|
| 399 |
+
|
| 400 |
+
### Colors Explanation:
|
| 401 |
+
|
| 402 |
+
- 🔵 **Blue (Client)**: User's browser, HLS player, client-side actions
|
| 403 |
+
- 🔴 **Red (Controller)**: NestJS controllers, HTTP endpoints, request handling
|
| 404 |
+
- 🟢 **Green (Service)**: Business logic, core functionality, data processing
|
| 405 |
+
- 🟠 **Orange (Wasabi)**: Wasabi S3 storage, cloud storage operations
|
| 406 |
+
- 🟣 **Purple (HLS)**: FFmpeg conversion, video processing, quality generation
|
| 407 |
+
- 🟡 **Yellow (Performance)**: Performance metrics, speed comparisons
|
| 408 |
+
- 🔷 **Teal (URL Rewriting)**: URL transformation logic, path rewriting
|
| 409 |
+
|
| 410 |
+
---
|
| 411 |
+
|
| 412 |
+
## 📝 Key Points in the Diagram
|
| 413 |
+
|
| 414 |
+
### 1. **Upload Flow** (Top Section)
|
| 415 |
+
|
| 416 |
+
- User uploads video → Controller receives → Service processes
|
| 417 |
+
- Async HLS conversion starts immediately (non-blocking)
|
| 418 |
+
- FFmpeg generates 4 qualities: 1080p, 720p, 480p, audio-only
|
| 419 |
+
- All files uploaded to Wasabi under `hls/uploadId/`
|
| 420 |
+
- Local files cleaned up after upload
|
| 421 |
+
|
| 422 |
+
### 2. **Master Playlist Flow** (Middle Section)
|
| 423 |
+
|
| 424 |
+
- Client requests master.m3u8
|
| 425 |
+
- Server fetches from Wasabi (internal S3 request)
|
| 426 |
+
- **URL Rewriting** converts relative paths to absolute URLs
|
| 427 |
+
- Client receives rewritten playlist with proxy URLs
|
| 428 |
+
|
| 429 |
+
### 3. **Quality Playlist Flow** (Middle-Lower Section)
|
| 430 |
+
|
| 431 |
+
- Client selects quality and requests quality playlist
|
| 432 |
+
- Server fetches and **automatically rewrites** segment URLs
|
| 433 |
+
- All .ts segment references converted to proxy URLs
|
| 434 |
+
|
| 435 |
+
### 4. **Segment Streaming Flow** (Lower Section)
|
| 436 |
+
|
| 437 |
+
- Client requests video segments in parallel
|
| 438 |
+
- Server streams directly from Wasabi (no buffering)
|
| 439 |
+
- **Direct piping** for real-time chunk delivery
|
| 440 |
+
- Client buffers and plays video smoothly
|
| 441 |
+
|
| 442 |
+
### 5. **Performance Comparison** (Bottom Section)
|
| 443 |
+
|
| 444 |
+
- **Local Testing**: Slow due to home internet upload speed (5-20 Mbps)
|
| 445 |
+
- **Production**: Fast due to datacenter speeds (1-10 Gbps, 1000x faster!)
|
| 446 |
+
|
| 447 |
+
---
|
| 448 |
+
|
| 449 |
+
## 🔑 Critical Components
|
| 450 |
+
|
| 451 |
+
### URL Rewriting (The Magic!)
|
| 452 |
+
|
| 453 |
+
```
|
| 454 |
+
Original: index_audio.m3u8
|
| 455 |
+
Rewritten: https://domain/wasabi/file?path=hls%2FuploadId%2Findex_audio.m3u8
|
| 456 |
+
|
| 457 |
+
Original: segment_720p_000.ts
|
| 458 |
+
Rewritten: https://domain/wasabi/file?path=hls%2FuploadId%2Fsegment_720p_000.ts
|
| 459 |
+
```
|
| 460 |
+
|
| 461 |
+
### Direct Piping (Performance!)
|
| 462 |
+
|
| 463 |
+
```javascript
|
| 464 |
+
// NOT this (buffering - slow):
|
| 465 |
+
const buffer = await downloadFromWasabi();
|
| 466 |
+
res.send(buffer);
|
| 467 |
+
|
| 468 |
+
// THIS (direct piping - fast):
|
| 469 |
+
wasabiStream.pipe(res);
|
| 470 |
+
```
|
| 471 |
+
|
| 472 |
+
---
|
| 473 |
+
|
| 474 |
+
## 🚀 How to Use This Diagram
|
| 475 |
+
|
| 476 |
+
1. **Understanding Upload**: Follow from top (User uploads) → HLS conversion → Wasabi upload
|
| 477 |
+
2. **Understanding Streaming**: Follow from middle (Client requests) → URL rewriting → Segment delivery
|
| 478 |
+
3. **Understanding Performance**: Compare left side (Local/Slow) vs right side (Production/Fast)
|
| 479 |
+
4. **Teaching Others**: Show how data flows from client → server → Wasabi → server → client
|
| 480 |
+
5. **Debugging**: Trace the exact path when video is not loading
|
| 481 |
+
|
| 482 |
+
---
|
| 483 |
+
|
| 484 |
+
## 📊 Statistics at a Glance
|
| 485 |
+
|
| 486 |
+
| Metric | Value |
|
| 487 |
+
| ----------------------------- | ------------------------------------ |
|
| 488 |
+
| **Video Qualities** | 4 (1080p, 720p, 480p, audio-only) |
|
| 489 |
+
| **Segment Duration** | 6 seconds per chunk |
|
| 490 |
+
| **Typical Segment Size** | 1-2 MB |
|
| 491 |
+
| **Playlists Generated** | 5 (.m3u8 files) |
|
| 492 |
+
| **Total Files** | 100-500+ (depending on video length) |
|
| 493 |
+
| **Upload Speed (Local)** | 5-20 Mbps |
|
| 494 |
+
| **Upload Speed (Production)** | 1000-10000 Mbps (100-1000x faster) |
|
| 495 |
+
| **Initial Load (Local)** | 4-6 seconds |
|
| 496 |
+
| **Initial Load (Production)** | 0.5-1 second |
|
| 497 |
+
|
| 498 |
+
---
|
| 499 |
+
|
| 500 |
+
## 💡 Why This Architecture Works
|
| 501 |
+
|
| 502 |
+
✅ **Wasabi Free Trial Compatible** - No public bucket needed
|
| 503 |
+
✅ **Secure** - No direct Wasabi URLs exposed
|
| 504 |
+
✅ **Scalable** - Direct piping keeps memory low
|
| 505 |
+
✅ **Fast in Production** - Datacenter-to-datacenter speeds
|
| 506 |
+
✅ **Flexible** - Can add auth, analytics, rate limiting later
|
| 507 |
+
✅ **Cost Effective** - No egress fees from Wasabi
|
| 508 |
+
|
| 509 |
+
---
|
| 510 |
+
|
| 511 |
+
## 🎬 End Result
|
| 512 |
+
|
| 513 |
+
User uploads video → Automatic HLS conversion → Stored in Wasabi → Anyone can play via HLS player using your server's proxy URLs → Smooth streaming in production! 🚀
|
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
export enum FileType {
|
| 2 |
+
IMAGE = 'IMAGE',
|
| 3 |
+
DOCUMENT = 'DOCUMENT',
|
| 4 |
+
VIDEO = 'VIDEO',
|
| 5 |
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { registerAs } from '@nestjs/config';
|
| 2 |
+
|
| 3 |
+
export default registerAs('wasabiStorage', () => ({
|
| 4 |
+
endpoint: process.env.WASABI_ENDPOINT,
|
| 5 |
+
region: process.env.WASABI_REGION || 'us-west-1',
|
| 6 |
+
accessKeyId: process.env.WASABI_ACCESS_KEY_ID,
|
| 7 |
+
secretAccessKey: process.env.WASABI_SECRET_ACCESS_KEY,
|
| 8 |
+
bucket: process.env.WASABI_BUCKET_NAME || 'streamflix-backed',
|
| 9 |
+
}));
|
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import {
|
| 2 |
+
Controller,
|
| 3 |
+
Get,
|
| 4 |
+
Delete,
|
| 5 |
+
Post,
|
| 6 |
+
Query,
|
| 7 |
+
HttpCode,
|
| 8 |
+
HttpStatus,
|
| 9 |
+
Res,
|
| 10 |
+
Header,
|
| 11 |
+
UseInterceptors,
|
| 12 |
+
UploadedFile,
|
| 13 |
+
BadRequestException,
|
| 14 |
+
} from '@nestjs/common';
|
| 15 |
+
import type { Response } from 'express';
|
| 16 |
+
import { FileInterceptor } from '@nestjs/platform-express';
|
| 17 |
+
import {
|
| 18 |
+
ApiTags,
|
| 19 |
+
ApiOperation,
|
| 20 |
+
ApiQuery,
|
| 21 |
+
ApiConsumes,
|
| 22 |
+
ApiBody,
|
| 23 |
+
} from '@nestjs/swagger';
|
| 24 |
+
import { WasabiStorageService } from './wasabi-storage.service';
|
| 25 |
+
import { FileType } from './enums/file-type.enum';
|
| 26 |
+
import { LogToDiscord } from 'src/shared/decorators/log-to-discord.decorator';
|
| 27 |
+
import { SkipApiToken } from 'src/shared/decorators/skip-api-token.decorator';
|
| 28 |
+
import { diskStorage } from 'multer';
|
| 29 |
+
import { extname } from 'path';
|
| 30 |
+
import * as fs from 'fs/promises';
|
| 31 |
+
|
| 32 |
+
@ApiTags('Upload: Wasabi Storage')
|
| 33 |
+
@Controller('wasabi')
|
| 34 |
+
@LogToDiscord()
|
| 35 |
+
export class WasabiStorageController {
|
| 36 |
+
constructor(private readonly wasabiStorageService: WasabiStorageService) {}
|
| 37 |
+
|
| 38 |
+
@Get('presigned-upload')
|
| 39 |
+
@ApiOperation({ summary: 'Get presigned upload URL' })
|
| 40 |
+
@ApiQuery({ name: 'fileName', required: true })
|
| 41 |
+
@ApiQuery({ name: 'fileType', enum: FileType, required: false })
|
| 42 |
+
async getPresignedUploadUrl(
|
| 43 |
+
@Query('fileName') fileName: string,
|
| 44 |
+
@Query('fileType') fileType?: FileType,
|
| 45 |
+
) {
|
| 46 |
+
return this.wasabiStorageService.getPresignedUploadUrl(fileName, fileType);
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
@Get('file')
|
| 50 |
+
@SkipApiToken()
|
| 51 |
+
@ApiOperation({
|
| 52 |
+
summary: 'Stream file from Wasabi (Proxy CDN endpoint)',
|
| 53 |
+
description:
|
| 54 |
+
'Acts as a proxy to stream files from Wasabi. Works without public bucket access. Automatically rewrites HLS playlists.',
|
| 55 |
+
})
|
| 56 |
+
@ApiQuery({
|
| 57 |
+
name: 'path',
|
| 58 |
+
required: true,
|
| 59 |
+
description: 'File path in bucket (e.g., videos/1234567890-file.mp4)',
|
| 60 |
+
})
|
| 61 |
+
@Header('Accept-Ranges', 'bytes')
|
| 62 |
+
async streamFile(
|
| 63 |
+
@Query('path') path: string,
|
| 64 |
+
@Res({ passthrough: false }) res: Response,
|
| 65 |
+
) {
|
| 66 |
+
try {
|
| 67 |
+
const result = await this.wasabiStorageService.streamFile(path);
|
| 68 |
+
|
| 69 |
+
// Set response headers
|
| 70 |
+
res.set({
|
| 71 |
+
'Content-Type': result.contentType,
|
| 72 |
+
'Content-Length': result.contentLength.toString(),
|
| 73 |
+
'Content-Disposition': `inline; filename="${result.fileName}"`,
|
| 74 |
+
'Cache-Control': 'public, max-age=2592000', // Cache for 30 days
|
| 75 |
+
'Access-Control-Allow-Origin': '*', // Allow CORS for video players
|
| 76 |
+
});
|
| 77 |
+
|
| 78 |
+
// If HLS playlist was rewritten, send the modified content as string
|
| 79 |
+
if (result.isRewritten && result.rewrittenContent) {
|
| 80 |
+
return res.send(result.rewrittenContent);
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
// Otherwise, pipe the stream directly to response
|
| 84 |
+
result.stream.pipe(res);
|
| 85 |
+
} catch (error) {
|
| 86 |
+
// Check if it's a 404 error (file not found)
|
| 87 |
+
const isNotFound =
|
| 88 |
+
error.status === 404 || error.status === HttpStatus.NOT_FOUND;
|
| 89 |
+
|
| 90 |
+
if (isNotFound) {
|
| 91 |
+
// Send beautiful 404 error page
|
| 92 |
+
res.status(404).send(this.wasabiStorageService.generate404Page(path));
|
| 93 |
+
} else {
|
| 94 |
+
// Other errors - send JSON
|
| 95 |
+
res.status(error.status || 500).json({
|
| 96 |
+
success: false,
|
| 97 |
+
message: error.message || 'Failed to stream file',
|
| 98 |
+
});
|
| 99 |
+
}
|
| 100 |
+
}
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
@Delete('by-url')
|
| 104 |
+
@HttpCode(HttpStatus.OK)
|
| 105 |
+
@ApiOperation({ summary: 'Delete file by CDN URL' })
|
| 106 |
+
async deleteFileByUrl(@Query('url') url: string) {
|
| 107 |
+
const result = await this.wasabiStorageService.deleteFileByUrl(url);
|
| 108 |
+
return {
|
| 109 |
+
message: 'File deleted successfully',
|
| 110 |
+
...result,
|
| 111 |
+
};
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
@Post('upload-video-hls')
|
| 115 |
+
@HttpCode(HttpStatus.OK)
|
| 116 |
+
@ApiOperation({
|
| 117 |
+
summary: 'Upload MP4 video, convert to HLS, and store in Wasabi',
|
| 118 |
+
description:
|
| 119 |
+
'Upload a video file. It will be converted to HLS format (1080p, 720p, 480p) and stored in Wasabi under hls/{uploadId}/',
|
| 120 |
+
})
|
| 121 |
+
@ApiConsumes('multipart/form-data')
|
| 122 |
+
@ApiBody({
|
| 123 |
+
schema: {
|
| 124 |
+
type: 'object',
|
| 125 |
+
properties: {
|
| 126 |
+
video: {
|
| 127 |
+
type: 'string',
|
| 128 |
+
format: 'binary',
|
| 129 |
+
description: 'Video file (MP4, AVI, MOV, etc.)',
|
| 130 |
+
},
|
| 131 |
+
},
|
| 132 |
+
},
|
| 133 |
+
})
|
| 134 |
+
@UseInterceptors(
|
| 135 |
+
FileInterceptor('video', {
|
| 136 |
+
storage: diskStorage({
|
| 137 |
+
destination: './resources/temp',
|
| 138 |
+
filename: (req, file, callback) => {
|
| 139 |
+
const uniqueSuffix =
|
| 140 |
+
Date.now() + '-' + Math.round(Math.random() * 1e9);
|
| 141 |
+
callback(null, `video-${uniqueSuffix}${extname(file.originalname)}`);
|
| 142 |
+
},
|
| 143 |
+
}),
|
| 144 |
+
fileFilter: (req, file, callback) => {
|
| 145 |
+
if (!file.mimetype.startsWith('video/')) {
|
| 146 |
+
return callback(
|
| 147 |
+
new BadRequestException('Only video files are allowed'),
|
| 148 |
+
false,
|
| 149 |
+
);
|
| 150 |
+
}
|
| 151 |
+
callback(null, true);
|
| 152 |
+
},
|
| 153 |
+
limits: {
|
| 154 |
+
fileSize: 500 * 1024 * 1024, // 500MB max
|
| 155 |
+
},
|
| 156 |
+
}),
|
| 157 |
+
)
|
| 158 |
+
async uploadVideoAndConvertToHLS(@UploadedFile() file: Express.Multer.File) {
|
| 159 |
+
if (!file) {
|
| 160 |
+
throw new BadRequestException('No video file provided');
|
| 161 |
+
}
|
| 162 |
+
try {
|
| 163 |
+
const { uploadId } = await this.wasabiStorageService.startVideoProcessing(
|
| 164 |
+
file.originalname,
|
| 165 |
+
file.size,
|
| 166 |
+
file.path,
|
| 167 |
+
);
|
| 168 |
+
|
| 169 |
+
return {
|
| 170 |
+
success: true,
|
| 171 |
+
message: 'Upload received. Conversion started and processing.',
|
| 172 |
+
data: {
|
| 173 |
+
uploadId,
|
| 174 |
+
fileName: file.originalname,
|
| 175 |
+
originalSize: file.size,
|
| 176 |
+
status: 'processing',
|
| 177 |
+
},
|
| 178 |
+
};
|
| 179 |
+
} catch (error) {
|
| 180 |
+
try {
|
| 181 |
+
await fs.unlink(file.path);
|
| 182 |
+
} catch {}
|
| 183 |
+
throw error;
|
| 184 |
+
}
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
@Get('hls')
|
| 188 |
+
@SkipApiToken()
|
| 189 |
+
@ApiOperation({
|
| 190 |
+
summary: 'Stream HLS master m3u8 playlist with rewritten URLs',
|
| 191 |
+
description:
|
| 192 |
+
'Serves the master.m3u8 file with rewritten URLs pointing to this server (not direct Wasabi)',
|
| 193 |
+
})
|
| 194 |
+
@ApiQuery({
|
| 195 |
+
name: 'uploadId',
|
| 196 |
+
required: true,
|
| 197 |
+
description: 'Upload ID from HLS conversion',
|
| 198 |
+
})
|
| 199 |
+
async streamHLS(@Query('uploadId') uploadId: string, @Res() res: Response) {
|
| 200 |
+
try {
|
| 201 |
+
const { m3u8Content, contentType } =
|
| 202 |
+
await this.wasabiStorageService.getRewrittenM3U8(uploadId);
|
| 203 |
+
|
| 204 |
+
res.set({
|
| 205 |
+
'Content-Type': contentType,
|
| 206 |
+
'Cache-Control': 'no-cache',
|
| 207 |
+
'Access-Control-Allow-Origin': '*',
|
| 208 |
+
});
|
| 209 |
+
|
| 210 |
+
res.send(m3u8Content);
|
| 211 |
+
} catch (error) {
|
| 212 |
+
res.status(error.status || 500).json({
|
| 213 |
+
success: false,
|
| 214 |
+
message: error.message || 'Failed to stream HLS',
|
| 215 |
+
});
|
| 216 |
+
}
|
| 217 |
+
}
|
| 218 |
+
|
| 219 |
+
@Delete('hls')
|
| 220 |
+
@HttpCode(HttpStatus.OK)
|
| 221 |
+
@ApiOperation({ summary: 'Delete HLS video files from Wasabi' })
|
| 222 |
+
@ApiQuery({ name: 'uploadId', required: true })
|
| 223 |
+
async deleteHLSVideo(@Query('uploadId') uploadId: string) {
|
| 224 |
+
if (!uploadId) {
|
| 225 |
+
throw new BadRequestException('uploadId is required');
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
const result =
|
| 229 |
+
await this.wasabiStorageService.deleteProcessedVideo(uploadId);
|
| 230 |
+
|
| 231 |
+
return {
|
| 232 |
+
success: true,
|
| 233 |
+
message: 'HLS video deleted successfully',
|
| 234 |
+
data: result,
|
| 235 |
+
};
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
+
@Get('hls/status')
|
| 239 |
+
@ApiOperation({ summary: 'Get HLS conversion status' })
|
| 240 |
+
@ApiQuery({ name: 'uploadId', required: true })
|
| 241 |
+
async getHLSStatus(@Query('uploadId') uploadId: string) {
|
| 242 |
+
if (!uploadId) {
|
| 243 |
+
throw new BadRequestException('uploadId is required');
|
| 244 |
+
}
|
| 245 |
+
const status = await this.wasabiStorageService.getUploadStatus(uploadId);
|
| 246 |
+
|
| 247 |
+
return {
|
| 248 |
+
success: true,
|
| 249 |
+
data: status,
|
| 250 |
+
};
|
| 251 |
+
}
|
| 252 |
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { Module } from '@nestjs/common';
|
| 2 |
+
import { ConfigModule } from '@nestjs/config';
|
| 3 |
+
import { WasabiStorageController } from './wasabi-storage.controller';
|
| 4 |
+
import { WasabiStorageService } from './wasabi-storage.service';
|
| 5 |
+
import wasabiStorageConfig from './wasabi-storage.config';
|
| 6 |
+
import { HlsConversionModule } from 'src/shared/modules/hls-conversion/hls-conversion.module';
|
| 7 |
+
|
| 8 |
+
@Module({
|
| 9 |
+
imports: [ConfigModule.forFeature(wasabiStorageConfig), HlsConversionModule],
|
| 10 |
+
controllers: [WasabiStorageController],
|
| 11 |
+
providers: [WasabiStorageService],
|
| 12 |
+
exports: [WasabiStorageService],
|
| 13 |
+
})
|
| 14 |
+
export class WasabiStorageModule {}
|
|
@@ -0,0 +1,624 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
|
| 2 |
+
import { ConfigService } from '@nestjs/config';
|
| 3 |
+
import {
|
| 4 |
+
S3Client,
|
| 5 |
+
PutObjectCommand,
|
| 6 |
+
DeleteObjectCommand,
|
| 7 |
+
ListObjectVersionsCommand,
|
| 8 |
+
GetObjectCommand,
|
| 9 |
+
ListObjectsV2Command,
|
| 10 |
+
} from '@aws-sdk/client-s3';
|
| 11 |
+
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
| 12 |
+
import { generate404ErrorPage } from 'src/shared/utils/error-pages/error-404.util';
|
| 13 |
+
import { HlsConversionService } from 'src/shared/modules/hls-conversion/hls-conversion.service';
|
| 14 |
+
import * as fs from 'fs/promises';
|
| 15 |
+
import * as path from 'path';
|
| 16 |
+
|
| 17 |
+
@Injectable()
|
| 18 |
+
export class WasabiStorageService {
|
| 19 |
+
private readonly logger = new Logger(WasabiStorageService.name);
|
| 20 |
+
private s3Client: S3Client;
|
| 21 |
+
private bucket: string;
|
| 22 |
+
private region: string;
|
| 23 |
+
private endpoint: string;
|
| 24 |
+
private cdnUrl: string;
|
| 25 |
+
|
| 26 |
+
constructor(
|
| 27 |
+
private configService: ConfigService,
|
| 28 |
+
private hlsConversionService: HlsConversionService,
|
| 29 |
+
) {
|
| 30 |
+
this.endpoint =
|
| 31 |
+
this.configService.get<string>('wasabiStorage.endpoint') || '';
|
| 32 |
+
this.region =
|
| 33 |
+
this.configService.get<string>('wasabiStorage.region') || 'us-west-1';
|
| 34 |
+
const accessKeyId =
|
| 35 |
+
this.configService.get<string>('wasabiStorage.accessKeyId') || '';
|
| 36 |
+
const secretAccessKey =
|
| 37 |
+
this.configService.get<string>('wasabiStorage.secretAccessKey') || '';
|
| 38 |
+
this.bucket =
|
| 39 |
+
this.configService.get<string>('wasabiStorage.bucket') ||
|
| 40 |
+
'streamflix-backed';
|
| 41 |
+
// Use APP_URL for proxy endpoint instead of direct Wasabi URL
|
| 42 |
+
const appUrl =
|
| 43 |
+
this.configService.get<string>('APP_URL') || 'http://localhost:3000';
|
| 44 |
+
this.cdnUrl = `${appUrl}/wasabi/file`;
|
| 45 |
+
|
| 46 |
+
this.s3Client = new S3Client({
|
| 47 |
+
region: this.region,
|
| 48 |
+
endpoint: this.endpoint,
|
| 49 |
+
credentials: {
|
| 50 |
+
accessKeyId,
|
| 51 |
+
secretAccessKey,
|
| 52 |
+
},
|
| 53 |
+
});
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
/**
|
| 57 |
+
* Get folder path based on file type
|
| 58 |
+
* @param fileType - The file type (IMAGE, DOCUMENT, VIDEO)
|
| 59 |
+
* @returns Folder path for the file type
|
| 60 |
+
*/
|
| 61 |
+
private getFolderPath(fileType: string): string {
|
| 62 |
+
const folderMap: Record<string, string> = {
|
| 63 |
+
IMAGE: 'images',
|
| 64 |
+
DOCUMENT: 'documents',
|
| 65 |
+
VIDEO: 'videos',
|
| 66 |
+
};
|
| 67 |
+
return folderMap[fileType.toUpperCase()] || 'others';
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
/**
|
| 71 |
+
* Generate presigned upload URL using S3
|
| 72 |
+
* @param fileName - The file name
|
| 73 |
+
* @param fileType - The file type (IMAGE, DOCUMENT, VIDEO)
|
| 74 |
+
* @returns Presigned upload URL and CDN file URL
|
| 75 |
+
*/
|
| 76 |
+
async getPresignedUploadUrl(fileName: string, fileType?: string) {
|
| 77 |
+
try {
|
| 78 |
+
// Determine folder based on file type
|
| 79 |
+
const folder = fileType ? this.getFolderPath(fileType) : 'others';
|
| 80 |
+
|
| 81 |
+
// Create full file path: folder/timestamp-filename
|
| 82 |
+
const timestamp = Date.now();
|
| 83 |
+
const filePath = `${folder}/${timestamp}-${fileName}`;
|
| 84 |
+
|
| 85 |
+
// Create PutObject command
|
| 86 |
+
const command = new PutObjectCommand({
|
| 87 |
+
Bucket: this.bucket,
|
| 88 |
+
Key: filePath,
|
| 89 |
+
});
|
| 90 |
+
|
| 91 |
+
// Generate presigned URL (valid for 1 hour)
|
| 92 |
+
const uploadUrl = await getSignedUrl(this.s3Client, command, {
|
| 93 |
+
expiresIn: 3600,
|
| 94 |
+
});
|
| 95 |
+
|
| 96 |
+
// Construct proxy URL: http://localhost:5119/wasabi/file?path=videos/123-file.mp4
|
| 97 |
+
const fileUrl = `${this.cdnUrl}?path=${encodeURIComponent(filePath)}`;
|
| 98 |
+
|
| 99 |
+
this.logger.log(
|
| 100 |
+
`Generated presigned upload URL for: ${filePath}${fileType ? ` (type: ${fileType})` : ''}`,
|
| 101 |
+
);
|
| 102 |
+
|
| 103 |
+
return {
|
| 104 |
+
uploadUrl,
|
| 105 |
+
fileUrl, // Proxy URL (works without public bucket access)
|
| 106 |
+
filePath,
|
| 107 |
+
expiresIn: 3600, // 1 hour in seconds
|
| 108 |
+
...(fileType && { fileType }),
|
| 109 |
+
};
|
| 110 |
+
} catch (error) {
|
| 111 |
+
this.logger.error('Failed to generate presigned upload URL', error);
|
| 112 |
+
throw error;
|
| 113 |
+
}
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
/**
|
| 117 |
+
* Delete a file from Wasabi using S3
|
| 118 |
+
* Handles both versioned and non-versioned files
|
| 119 |
+
* @param filePath - The file path in the bucket
|
| 120 |
+
* @returns Deletion result
|
| 121 |
+
*/
|
| 122 |
+
async deleteFile(filePath: string): Promise<any> {
|
| 123 |
+
try {
|
| 124 |
+
// First, try to list all versions (handles versioned files)
|
| 125 |
+
const listCommand = new ListObjectVersionsCommand({
|
| 126 |
+
Bucket: this.bucket,
|
| 127 |
+
Prefix: filePath,
|
| 128 |
+
});
|
| 129 |
+
|
| 130 |
+
const listResult = await this.s3Client.send(listCommand);
|
| 131 |
+
|
| 132 |
+
const deletionPromises = [] as Array<Promise<any>>;
|
| 133 |
+
let versionsFound = false;
|
| 134 |
+
|
| 135 |
+
// Delete all versions if they exist
|
| 136 |
+
if (listResult.Versions && listResult.Versions.length > 0) {
|
| 137 |
+
for (const version of listResult.Versions) {
|
| 138 |
+
if (version.Key === filePath && version.VersionId) {
|
| 139 |
+
versionsFound = true;
|
| 140 |
+
const deleteCommand = new DeleteObjectCommand({
|
| 141 |
+
Bucket: this.bucket,
|
| 142 |
+
Key: version.Key,
|
| 143 |
+
VersionId: version.VersionId,
|
| 144 |
+
});
|
| 145 |
+
deletionPromises.push(this.s3Client.send(deleteCommand));
|
| 146 |
+
}
|
| 147 |
+
}
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
// Delete all delete markers if they exist
|
| 151 |
+
if (listResult.DeleteMarkers && listResult.DeleteMarkers.length > 0) {
|
| 152 |
+
for (const marker of listResult.DeleteMarkers) {
|
| 153 |
+
if (marker.Key === filePath && marker.VersionId) {
|
| 154 |
+
versionsFound = true;
|
| 155 |
+
const deleteCommand = new DeleteObjectCommand({
|
| 156 |
+
Bucket: this.bucket,
|
| 157 |
+
Key: marker.Key,
|
| 158 |
+
VersionId: marker.VersionId,
|
| 159 |
+
});
|
| 160 |
+
deletionPromises.push(this.s3Client.send(deleteCommand));
|
| 161 |
+
}
|
| 162 |
+
}
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
// If no versions found, do a simple delete (non-versioned file)
|
| 166 |
+
if (!versionsFound) {
|
| 167 |
+
const command = new DeleteObjectCommand({
|
| 168 |
+
Bucket: this.bucket,
|
| 169 |
+
Key: filePath,
|
| 170 |
+
});
|
| 171 |
+
const result = await this.s3Client.send(command);
|
| 172 |
+
|
| 173 |
+
this.logger.log(`Successfully deleted file: ${filePath}`);
|
| 174 |
+
return {
|
| 175 |
+
success: true,
|
| 176 |
+
deletedPath: filePath,
|
| 177 |
+
result,
|
| 178 |
+
};
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
// Execute all version deletions
|
| 182 |
+
const results = await Promise.all(deletionPromises);
|
| 183 |
+
|
| 184 |
+
this.logger.log(
|
| 185 |
+
`Successfully deleted file and all versions: ${filePath} (${deletionPromises.length} versions)`,
|
| 186 |
+
);
|
| 187 |
+
|
| 188 |
+
return {
|
| 189 |
+
success: true,
|
| 190 |
+
deletedPath: filePath,
|
| 191 |
+
versionsDeleted: deletionPromises.length,
|
| 192 |
+
results,
|
| 193 |
+
};
|
| 194 |
+
} catch (error) {
|
| 195 |
+
this.logger.error(`Failed to delete file: ${filePath}`, error);
|
| 196 |
+
throw new NotFoundException('File not found or already deleted');
|
| 197 |
+
}
|
| 198 |
+
}
|
| 199 |
+
|
| 200 |
+
/**
|
| 201 |
+
* Delete a file by CDN URL
|
| 202 |
+
* @param url - The CDN URL
|
| 203 |
+
* @returns Deletion result
|
| 204 |
+
*/
|
| 205 |
+
async deleteFileByUrl(url: string): Promise<any> {
|
| 206 |
+
try {
|
| 207 |
+
// Extract file path from URL
|
| 208 |
+
const filePath = this.extractFilePathFromUrl(url);
|
| 209 |
+
|
| 210 |
+
if (!filePath) {
|
| 211 |
+
throw new NotFoundException('Could not extract file path from URL');
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
// Delete from Wasabi
|
| 215 |
+
const result = await this.deleteFile(filePath);
|
| 216 |
+
|
| 217 |
+
this.logger.log(`Successfully deleted file from URL: ${url}`);
|
| 218 |
+
|
| 219 |
+
return {
|
| 220 |
+
...result,
|
| 221 |
+
deletedUrl: url,
|
| 222 |
+
};
|
| 223 |
+
} catch (error) {
|
| 224 |
+
if (error instanceof NotFoundException) {
|
| 225 |
+
throw error;
|
| 226 |
+
}
|
| 227 |
+
this.logger.error(`Failed to delete file by URL: ${url}`, error);
|
| 228 |
+
throw new NotFoundException('File not found or already deleted');
|
| 229 |
+
}
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
/**
|
| 233 |
+
* Extract file path from CDN URL
|
| 234 |
+
* @param url - The CDN URL (proxy URL format)
|
| 235 |
+
* @returns File path
|
| 236 |
+
*/
|
| 237 |
+
private extractFilePathFromUrl(url: string): string | null {
|
| 238 |
+
try {
|
| 239 |
+
// Match proxy URL: http://localhost:5119/wasabi/file?path=videos/123-file.mp4
|
| 240 |
+
const urlObj = new URL(url);
|
| 241 |
+
const path = urlObj.searchParams.get('path');
|
| 242 |
+
if (path) return path;
|
| 243 |
+
|
| 244 |
+
// Fallback: Match old format
|
| 245 |
+
const match = url.match(/^https?:\/\/[^/]+\/(.+)$/);
|
| 246 |
+
return match ? match[1] : null;
|
| 247 |
+
} catch {
|
| 248 |
+
return null;
|
| 249 |
+
}
|
| 250 |
+
}
|
| 251 |
+
|
| 252 |
+
/**
|
| 253 |
+
* Stream file from Wasabi (for proxy endpoint)
|
| 254 |
+
* @param filePath - The file path in the bucket
|
| 255 |
+
* @returns File stream and metadata
|
| 256 |
+
*/
|
| 257 |
+
async streamFile(filePath: string): Promise<{
|
| 258 |
+
stream: any;
|
| 259 |
+
contentType: string;
|
| 260 |
+
contentLength: number;
|
| 261 |
+
fileName: string;
|
| 262 |
+
isRewritten?: boolean;
|
| 263 |
+
rewrittenContent?: string;
|
| 264 |
+
}> {
|
| 265 |
+
try {
|
| 266 |
+
const command = new GetObjectCommand({
|
| 267 |
+
Bucket: this.bucket,
|
| 268 |
+
Key: filePath,
|
| 269 |
+
});
|
| 270 |
+
|
| 271 |
+
const result = await this.s3Client.send(command);
|
| 272 |
+
|
| 273 |
+
if (!result.Body) {
|
| 274 |
+
throw new NotFoundException('File not found');
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
// Extract filename from path
|
| 278 |
+
const fileName = filePath.split('/').pop() || 'file';
|
| 279 |
+
|
| 280 |
+
// Check if this is an HLS playlist file that needs URL rewriting
|
| 281 |
+
const isHlsPlaylist =
|
| 282 |
+
filePath.includes('hls/') && fileName.endsWith('.m3u8');
|
| 283 |
+
|
| 284 |
+
if (isHlsPlaylist) {
|
| 285 |
+
// Read the m3u8 content
|
| 286 |
+
const chunks: Uint8Array[] = [];
|
| 287 |
+
for await (const chunk of result.Body as any) {
|
| 288 |
+
chunks.push(chunk);
|
| 289 |
+
}
|
| 290 |
+
const originalContent = Buffer.concat(chunks).toString('utf-8');
|
| 291 |
+
|
| 292 |
+
// Extract uploadId from path: hls/uploadId/filename
|
| 293 |
+
const pathParts = filePath.split('/');
|
| 294 |
+
const uploadId = pathParts[1];
|
| 295 |
+
|
| 296 |
+
// Rewrite URLs in the playlist
|
| 297 |
+
const rewrittenContent = originalContent
|
| 298 |
+
.split('\n')
|
| 299 |
+
.map((line) => {
|
| 300 |
+
// Handle URIs inside quotes like: URI="index_audio.m3u8"
|
| 301 |
+
if (
|
| 302 |
+
line.includes('URI="') &&
|
| 303 |
+
(line.includes('.m3u8"') || line.includes('.ts"'))
|
| 304 |
+
) {
|
| 305 |
+
return line.replace(
|
| 306 |
+
/URI="([^"]+\.(m3u8|ts))"/g,
|
| 307 |
+
(match, filename) => {
|
| 308 |
+
return `URI="${this.cdnUrl}?path=${encodeURIComponent(`hls/${uploadId}/${filename}`)}"`;
|
| 309 |
+
},
|
| 310 |
+
);
|
| 311 |
+
}
|
| 312 |
+
// Handle standalone .m3u8 and .ts file references on their own line
|
| 313 |
+
if (line.trim().endsWith('.m3u8') || line.trim().endsWith('.ts')) {
|
| 314 |
+
const filename = line.trim();
|
| 315 |
+
return `${this.cdnUrl}?path=${encodeURIComponent(`hls/${uploadId}/${filename}`)}`;
|
| 316 |
+
}
|
| 317 |
+
return line;
|
| 318 |
+
})
|
| 319 |
+
.join('\n');
|
| 320 |
+
|
| 321 |
+
this.logger.log(`Streaming and rewriting HLS playlist: ${filePath}`);
|
| 322 |
+
|
| 323 |
+
return {
|
| 324 |
+
stream: null,
|
| 325 |
+
contentType: 'application/vnd.apple.mpegurl',
|
| 326 |
+
contentLength: Buffer.byteLength(rewrittenContent, 'utf-8'),
|
| 327 |
+
fileName,
|
| 328 |
+
isRewritten: true,
|
| 329 |
+
rewrittenContent,
|
| 330 |
+
};
|
| 331 |
+
}
|
| 332 |
+
|
| 333 |
+
this.logger.log(`Streaming file: ${filePath}`);
|
| 334 |
+
|
| 335 |
+
return {
|
| 336 |
+
stream: result.Body,
|
| 337 |
+
contentType: result.ContentType || 'application/octet-stream',
|
| 338 |
+
contentLength: result.ContentLength || 0,
|
| 339 |
+
fileName,
|
| 340 |
+
};
|
| 341 |
+
} catch (error) {
|
| 342 |
+
if (
|
| 343 |
+
error.Code === 'NoSuchKey' ||
|
| 344 |
+
error.$metadata?.httpStatusCode === 404
|
| 345 |
+
) {
|
| 346 |
+
this.logger.warn(`File not found in Wasabi: ${filePath}`);
|
| 347 |
+
throw new NotFoundException('File not found');
|
| 348 |
+
}
|
| 349 |
+
this.logger.error(`Failed to stream file: ${filePath}`, error);
|
| 350 |
+
throw new NotFoundException('File not found or access denied');
|
| 351 |
+
}
|
| 352 |
+
}
|
| 353 |
+
|
| 354 |
+
/**
|
| 355 |
+
* Generate 404 error page for missing files
|
| 356 |
+
* @param filePath - The file path that was not found
|
| 357 |
+
* @returns HTML error page
|
| 358 |
+
*/
|
| 359 |
+
generate404Page(filePath: string): string {
|
| 360 |
+
return generate404ErrorPage({
|
| 361 |
+
fileId: filePath,
|
| 362 |
+
title: 'Oops!',
|
| 363 |
+
message: 'File Not Found',
|
| 364 |
+
description:
|
| 365 |
+
'The file you are looking for has been deleted or never existed. Please check the URL and try again.',
|
| 366 |
+
illustrationSrc: '/box/error-illustration',
|
| 367 |
+
});
|
| 368 |
+
}
|
| 369 |
+
|
| 370 |
+
/**
|
| 371 |
+
* Start HLS video processing
|
| 372 |
+
* @param originalName - Original video filename
|
| 373 |
+
* @param size - File size in bytes
|
| 374 |
+
* @param tempFilePath - Temporary file path
|
| 375 |
+
* @returns Upload ID
|
| 376 |
+
*/
|
| 377 |
+
async startVideoProcessing(
|
| 378 |
+
originalName: string,
|
| 379 |
+
size: number,
|
| 380 |
+
tempFilePath: string,
|
| 381 |
+
): Promise<{ uploadId: string }> {
|
| 382 |
+
const uploadId = `${Date.now()}-${Math.random().toString(36).substring(7)}`;
|
| 383 |
+
|
| 384 |
+
this.logger.log(`Starting HLS conversion for uploadId: ${uploadId}`);
|
| 385 |
+
|
| 386 |
+
// Start async processing (don't await)
|
| 387 |
+
this.processVideoAsync(uploadId, originalName, size, tempFilePath);
|
| 388 |
+
|
| 389 |
+
return { uploadId };
|
| 390 |
+
}
|
| 391 |
+
|
| 392 |
+
/**
|
| 393 |
+
* Process video asynchronously
|
| 394 |
+
*/
|
| 395 |
+
private async processVideoAsync(
|
| 396 |
+
uploadId: string,
|
| 397 |
+
originalName: string,
|
| 398 |
+
size: number,
|
| 399 |
+
tempFilePath: string,
|
| 400 |
+
): Promise<void> {
|
| 401 |
+
try {
|
| 402 |
+
// Convert to HLS (using existing hls-conversion module) - adaptive HLS with 480p, 720p, 1080p
|
| 403 |
+
const adaptive = await (
|
| 404 |
+
this.hlsConversionService as any
|
| 405 |
+
).convertVideoToAdaptiveHLS(uploadId, tempFilePath);
|
| 406 |
+
|
| 407 |
+
// Upload all HLS files to Wasabi
|
| 408 |
+
await this.uploadHLSFilesToWasabi(uploadId, adaptive);
|
| 409 |
+
|
| 410 |
+
// Clean up local files
|
| 411 |
+
await this.hlsConversionService.cleanupFiles(uploadId);
|
| 412 |
+
await fs.unlink(tempFilePath);
|
| 413 |
+
|
| 414 |
+
this.logger.log(`HLS processing completed for uploadId: ${uploadId}`);
|
| 415 |
+
} catch (error) {
|
| 416 |
+
this.logger.error(
|
| 417 |
+
`HLS processing failed for uploadId: ${uploadId}`,
|
| 418 |
+
error,
|
| 419 |
+
);
|
| 420 |
+
throw error;
|
| 421 |
+
}
|
| 422 |
+
}
|
| 423 |
+
|
| 424 |
+
/**
|
| 425 |
+
* Upload HLS files to Wasabi
|
| 426 |
+
*/
|
| 427 |
+
private async uploadHLSFilesToWasabi(
|
| 428 |
+
uploadId: string,
|
| 429 |
+
adaptive: any,
|
| 430 |
+
): Promise<void> {
|
| 431 |
+
// Upload master.m3u8
|
| 432 |
+
const masterContent = await fs.readFile(adaptive.masterM3u8Path, 'utf-8');
|
| 433 |
+
await this.uploadFileToWasabi(
|
| 434 |
+
`hls/${uploadId}/master.m3u8`,
|
| 435 |
+
Buffer.from(masterContent),
|
| 436 |
+
'application/vnd.apple.mpegurl',
|
| 437 |
+
);
|
| 438 |
+
|
| 439 |
+
// Upload all rendition files (480p, 720p, 1080p, audio-only)
|
| 440 |
+
for (const rendition of adaptive.renditions) {
|
| 441 |
+
// Upload rendition playlist m3u8
|
| 442 |
+
const playlistContent = await fs.readFile(rendition.m3u8Path, 'utf-8');
|
| 443 |
+
await this.uploadFileToWasabi(
|
| 444 |
+
`hls/${uploadId}/${path.basename(rendition.m3u8Path)}`,
|
| 445 |
+
Buffer.from(playlistContent),
|
| 446 |
+
'application/vnd.apple.mpegurl',
|
| 447 |
+
);
|
| 448 |
+
|
| 449 |
+
// Upload all TS segments for this rendition
|
| 450 |
+
for (const segmentPath of rendition.segmentPaths) {
|
| 451 |
+
const segmentBuffer = await fs.readFile(segmentPath);
|
| 452 |
+
await this.uploadFileToWasabi(
|
| 453 |
+
`hls/${uploadId}/${path.basename(segmentPath)}`,
|
| 454 |
+
segmentBuffer,
|
| 455 |
+
'video/mp2t',
|
| 456 |
+
);
|
| 457 |
+
}
|
| 458 |
+
}
|
| 459 |
+
|
| 460 |
+
this.logger.log(`Uploaded all HLS files for uploadId: ${uploadId}`);
|
| 461 |
+
}
|
| 462 |
+
|
| 463 |
+
/**
|
| 464 |
+
* Upload file buffer to Wasabi
|
| 465 |
+
*/
|
| 466 |
+
private async uploadFileToWasabi(
|
| 467 |
+
key: string,
|
| 468 |
+
buffer: Buffer,
|
| 469 |
+
contentType: string,
|
| 470 |
+
): Promise<void> {
|
| 471 |
+
const command = new PutObjectCommand({
|
| 472 |
+
Bucket: this.bucket,
|
| 473 |
+
Key: key,
|
| 474 |
+
Body: buffer,
|
| 475 |
+
ContentType: contentType,
|
| 476 |
+
});
|
| 477 |
+
|
| 478 |
+
await this.s3Client.send(command);
|
| 479 |
+
}
|
| 480 |
+
|
| 481 |
+
/**
|
| 482 |
+
* Get rewritten m3u8 file with proxy URLs
|
| 483 |
+
* @param uploadId - Upload ID
|
| 484 |
+
* @returns M3U8 content with rewritten URLs
|
| 485 |
+
*/
|
| 486 |
+
async getRewrittenM3U8(
|
| 487 |
+
uploadId: string,
|
| 488 |
+
): Promise<{ m3u8Content: string; contentType: string }> {
|
| 489 |
+
try {
|
| 490 |
+
// Fetch master.m3u8 from Wasabi
|
| 491 |
+
const command = new GetObjectCommand({
|
| 492 |
+
Bucket: this.bucket,
|
| 493 |
+
Key: `hls/${uploadId}/master.m3u8`,
|
| 494 |
+
});
|
| 495 |
+
|
| 496 |
+
const result = await this.s3Client.send(command);
|
| 497 |
+
|
| 498 |
+
if (!result.Body) {
|
| 499 |
+
throw new NotFoundException('M3U8 file not found');
|
| 500 |
+
}
|
| 501 |
+
|
| 502 |
+
// Convert stream to string
|
| 503 |
+
const chunks: Uint8Array[] = [];
|
| 504 |
+
for await (const chunk of result.Body as any) {
|
| 505 |
+
chunks.push(chunk);
|
| 506 |
+
}
|
| 507 |
+
const originalContent = Buffer.concat(chunks).toString('utf-8');
|
| 508 |
+
|
| 509 |
+
// Rewrite URLs to point to our proxy
|
| 510 |
+
const appUrl =
|
| 511 |
+
this.configService.get<string>('APP_URL') || 'http://localhost:3000';
|
| 512 |
+
|
| 513 |
+
const rewrittenContent = originalContent
|
| 514 |
+
.split('\n')
|
| 515 |
+
.map((line) => {
|
| 516 |
+
// Handle URIs inside quotes like: URI="index_audio.m3u8"
|
| 517 |
+
if (
|
| 518 |
+
line.includes('URI="') &&
|
| 519 |
+
(line.includes('.m3u8"') || line.includes('.ts"'))
|
| 520 |
+
) {
|
| 521 |
+
return line.replace(
|
| 522 |
+
/URI="([^"]+\.(m3u8|ts))"/g,
|
| 523 |
+
(match, filename) => {
|
| 524 |
+
return `URI="${appUrl}/wasabi/file?path=${encodeURIComponent(`hls/${uploadId}/${filename}`)}"`;
|
| 525 |
+
},
|
| 526 |
+
);
|
| 527 |
+
}
|
| 528 |
+
// Handle standalone .m3u8 and .ts file references on their own line
|
| 529 |
+
if (line.trim().endsWith('.m3u8') || line.trim().endsWith('.ts')) {
|
| 530 |
+
const filename = line.trim();
|
| 531 |
+
return `${appUrl}/wasabi/file?path=${encodeURIComponent(`hls/${uploadId}/${filename}`)}`;
|
| 532 |
+
}
|
| 533 |
+
return line;
|
| 534 |
+
})
|
| 535 |
+
.join('\n');
|
| 536 |
+
|
| 537 |
+
this.logger.log(`Served rewritten m3u8 for uploadId: ${uploadId}`);
|
| 538 |
+
|
| 539 |
+
return {
|
| 540 |
+
m3u8Content: rewrittenContent,
|
| 541 |
+
contentType: 'application/vnd.apple.mpegurl',
|
| 542 |
+
};
|
| 543 |
+
} catch (error) {
|
| 544 |
+
if (
|
| 545 |
+
error.Code === 'NoSuchKey' ||
|
| 546 |
+
error.$metadata?.httpStatusCode === 404
|
| 547 |
+
) {
|
| 548 |
+
throw new NotFoundException('HLS video not found');
|
| 549 |
+
}
|
| 550 |
+
this.logger.error(`Failed to get m3u8 for uploadId: ${uploadId}`, error);
|
| 551 |
+
throw error;
|
| 552 |
+
}
|
| 553 |
+
}
|
| 554 |
+
|
| 555 |
+
/**
|
| 556 |
+
* Delete processed HLS video
|
| 557 |
+
*/
|
| 558 |
+
async deleteProcessedVideo(uploadId: string): Promise<any> {
|
| 559 |
+
try {
|
| 560 |
+
// List all objects with prefix
|
| 561 |
+
const listCommand = new ListObjectsV2Command({
|
| 562 |
+
Bucket: this.bucket,
|
| 563 |
+
Prefix: `hls/${uploadId}/`,
|
| 564 |
+
});
|
| 565 |
+
|
| 566 |
+
const listResult = await this.s3Client.send(listCommand);
|
| 567 |
+
|
| 568 |
+
if (!listResult.Contents || listResult.Contents.length === 0) {
|
| 569 |
+
throw new NotFoundException('HLS video not found');
|
| 570 |
+
}
|
| 571 |
+
|
| 572 |
+
// Delete all files
|
| 573 |
+
const deletePromises = listResult.Contents.map((obj) =>
|
| 574 |
+
this.deleteFile(obj.Key!),
|
| 575 |
+
);
|
| 576 |
+
|
| 577 |
+
await Promise.all(deletePromises);
|
| 578 |
+
|
| 579 |
+
this.logger.log(`Deleted HLS video: ${uploadId}`);
|
| 580 |
+
|
| 581 |
+
return {
|
| 582 |
+
success: true,
|
| 583 |
+
uploadId,
|
| 584 |
+
filesDeleted: listResult.Contents.length,
|
| 585 |
+
};
|
| 586 |
+
} catch (error) {
|
| 587 |
+
this.logger.error(`Failed to delete HLS video: ${uploadId}`, error);
|
| 588 |
+
throw error;
|
| 589 |
+
}
|
| 590 |
+
}
|
| 591 |
+
|
| 592 |
+
/**
|
| 593 |
+
* Get upload status (placeholder for future database integration)
|
| 594 |
+
*/
|
| 595 |
+
async getUploadStatus(uploadId: string): Promise<any> {
|
| 596 |
+
try {
|
| 597 |
+
// Check if files exist in Wasabi
|
| 598 |
+
const listCommand = new ListObjectsV2Command({
|
| 599 |
+
Bucket: this.bucket,
|
| 600 |
+
Prefix: `hls/${uploadId}/`,
|
| 601 |
+
MaxKeys: 1,
|
| 602 |
+
});
|
| 603 |
+
|
| 604 |
+
const listResult = await this.s3Client.send(listCommand);
|
| 605 |
+
|
| 606 |
+
const exists = listResult.Contents && listResult.Contents.length > 0;
|
| 607 |
+
|
| 608 |
+
return {
|
| 609 |
+
uploadId,
|
| 610 |
+
status: exists ? 'completed' : 'not_found',
|
| 611 |
+
m3u8Url: exists
|
| 612 |
+
? `${this.configService.get<string>('APP_URL') || 'http://localhost:3000'}/wasabi/hls?uploadId=${uploadId}`
|
| 613 |
+
: null,
|
| 614 |
+
};
|
| 615 |
+
} catch (error) {
|
| 616 |
+
this.logger.error(`Failed to get upload status: ${uploadId}`, error);
|
| 617 |
+
return {
|
| 618 |
+
uploadId,
|
| 619 |
+
status: 'error',
|
| 620 |
+
m3u8Url: null,
|
| 621 |
+
};
|
| 622 |
+
}
|
| 623 |
+
}
|
| 624 |
+
}
|