Spaces:
Runtime error
Runtime error
Akshar2325 commited on
Commit ·
5d53367
1
Parent(s): 51c6e04
feat(idrive-e2-storage): add idrive e2 storage integration
Browse files- Add IDrive e2 storage module with S3-compatible API integration
- Create file-type enum for categorizing uploads (IMAGE, DOCUMENT, VIDEO)
- Implement presigned URL generation for secure file uploads and downloads
- Add file deletion functionality with URL-based file removal
- Create storage configuration with environment variables for endpoint, region, access key, secret key, and bucket
- Add controller endpoints for presigned upload/download URLs and file deletion
- Implement automatic file organization by type into separate folders
- Add comprehensive error handling and logging throughout the service
- Update environment example file with IDrive e2 credentials configuration
- Integrate IDrive e2 module into main upload module exports
- .env.example +7 -0
- src/shared/modules/upload/idrive-e2-storage/enums/file-type.enum.ts +5 -0
- src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.config.ts +9 -0
- src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.controller.ts +62 -0
- src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.module.ts +13 -0
- src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.service.ts +193 -0
- src/shared/modules/upload/upload.module.ts +2 -0
.env.example
CHANGED
|
@@ -103,3 +103,10 @@ RUSTFS_ACCESS_KEY="your-rustfs-access-key"
|
|
| 103 |
RUSTFS_SECRET_KEY="your-rustfs-secret-key"
|
| 104 |
RUSTFS_BUCKET="streamflix"
|
| 105 |
RUSTFS_REGION="us-east-1"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
RUSTFS_SECRET_KEY="your-rustfs-secret-key"
|
| 104 |
RUSTFS_BUCKET="streamflix"
|
| 105 |
RUSTFS_REGION="us-east-1"
|
| 106 |
+
|
| 107 |
+
# IDrive e2 Storage Credentials
|
| 108 |
+
IDRIVE_E2_ENDPOINT="https://s3.ap-southeast-1.idrivee2.com"
|
| 109 |
+
IDRIVE_E2_REGION="ap-southeast-1"
|
| 110 |
+
IDRIVE_E2_ACCESS_KEY="your-idrive-access-key"
|
| 111 |
+
IDRIVE_E2_SECRET_KEY="your-idrive-secret-key"
|
| 112 |
+
IDRIVE_E2_BUCKET="streamflix"
|
src/shared/modules/upload/idrive-e2-storage/enums/file-type.enum.ts
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
export enum FileType {
|
| 2 |
+
IMAGE = 'IMAGE',
|
| 3 |
+
DOCUMENT = 'DOCUMENT',
|
| 4 |
+
VIDEO = 'VIDEO',
|
| 5 |
+
}
|
src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.config.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { registerAs } from '@nestjs/config';
|
| 2 |
+
|
| 3 |
+
export default registerAs('idriveE2Storage', () => ({
|
| 4 |
+
endpoint: process.env.IDRIVE_E2_ENDPOINT,
|
| 5 |
+
region: process.env.IDRIVE_E2_REGION || 'ap-southeast-1',
|
| 6 |
+
accessKey: process.env.IDRIVE_E2_ACCESS_KEY,
|
| 7 |
+
secretKey: process.env.IDRIVE_E2_SECRET_KEY,
|
| 8 |
+
bucket: process.env.IDRIVE_E2_BUCKET || 'streamflix',
|
| 9 |
+
}));
|
src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.controller.ts
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import {
|
| 2 |
+
Controller,
|
| 3 |
+
Get,
|
| 4 |
+
Delete,
|
| 5 |
+
Query,
|
| 6 |
+
HttpCode,
|
| 7 |
+
HttpStatus,
|
| 8 |
+
} from '@nestjs/common';
|
| 9 |
+
import { ApiTags, ApiOperation, ApiQuery } from '@nestjs/swagger';
|
| 10 |
+
import { IdriveE2StorageService } from './idrive-e2-storage.service';
|
| 11 |
+
import { FileType } from './enums/file-type.enum';
|
| 12 |
+
|
| 13 |
+
@ApiTags('Upload: IDrive e2 Storage')
|
| 14 |
+
@Controller('idrive-e2')
|
| 15 |
+
export class IdriveE2StorageController {
|
| 16 |
+
constructor(
|
| 17 |
+
private readonly idriveE2StorageService: IdriveE2StorageService,
|
| 18 |
+
) {}
|
| 19 |
+
|
| 20 |
+
@Get('presigned-upload')
|
| 21 |
+
@ApiOperation({ summary: 'Get presigned upload URL' })
|
| 22 |
+
@ApiQuery({ name: 'fileName', required: true })
|
| 23 |
+
@ApiQuery({ name: 'fileType', enum: FileType, required: false })
|
| 24 |
+
async getPresignedUploadUrl(
|
| 25 |
+
@Query('fileName') fileName: string,
|
| 26 |
+
@Query('fileType') fileType?: FileType,
|
| 27 |
+
) {
|
| 28 |
+
return this.idriveE2StorageService.getPresignedUploadUrl(
|
| 29 |
+
fileName,
|
| 30 |
+
fileType,
|
| 31 |
+
);
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
@Get('presigned-download')
|
| 35 |
+
@ApiOperation({ summary: 'Get presigned download URL for existing file' })
|
| 36 |
+
@ApiQuery({ name: 'filePath', required: true })
|
| 37 |
+
@ApiQuery({
|
| 38 |
+
name: 'expiresIn',
|
| 39 |
+
required: false,
|
| 40 |
+
description: 'Expiration time in seconds (default: 604800)',
|
| 41 |
+
})
|
| 42 |
+
async getPresignedDownloadUrl(
|
| 43 |
+
@Query('filePath') filePath: string,
|
| 44 |
+
@Query('expiresIn') expiresIn?: number,
|
| 45 |
+
) {
|
| 46 |
+
return this.idriveE2StorageService.getPresignedDownloadUrl(
|
| 47 |
+
filePath,
|
| 48 |
+
expiresIn,
|
| 49 |
+
);
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
@Delete('by-url')
|
| 53 |
+
@HttpCode(HttpStatus.OK)
|
| 54 |
+
@ApiOperation({ summary: 'Delete file by URL' })
|
| 55 |
+
async deleteFileByUrl(@Query('url') url: string) {
|
| 56 |
+
const result = await this.idriveE2StorageService.deleteFileByUrl(url);
|
| 57 |
+
return {
|
| 58 |
+
message: 'File deleted successfully',
|
| 59 |
+
...result,
|
| 60 |
+
};
|
| 61 |
+
}
|
| 62 |
+
}
|
src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.module.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { Module } from '@nestjs/common';
|
| 2 |
+
import { ConfigModule } from '@nestjs/config';
|
| 3 |
+
import { IdriveE2StorageController } from './idrive-e2-storage.controller';
|
| 4 |
+
import { IdriveE2StorageService } from './idrive-e2-storage.service';
|
| 5 |
+
import idriveE2StorageConfig from './idrive-e2-storage.config';
|
| 6 |
+
|
| 7 |
+
@Module({
|
| 8 |
+
imports: [ConfigModule.forFeature(idriveE2StorageConfig)],
|
| 9 |
+
controllers: [IdriveE2StorageController],
|
| 10 |
+
providers: [IdriveE2StorageService],
|
| 11 |
+
exports: [IdriveE2StorageService],
|
| 12 |
+
})
|
| 13 |
+
export class IdriveE2StorageModule {}
|
src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.service.ts
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
|
| 2 |
+
import { ConfigService } from '@nestjs/config';
|
| 3 |
+
import {
|
| 4 |
+
S3Client,
|
| 5 |
+
PutObjectCommand,
|
| 6 |
+
GetObjectCommand,
|
| 7 |
+
DeleteObjectCommand,
|
| 8 |
+
} from '@aws-sdk/client-s3';
|
| 9 |
+
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
| 10 |
+
|
| 11 |
+
@Injectable()
|
| 12 |
+
export class IdriveE2StorageService {
|
| 13 |
+
private readonly logger = new Logger(IdriveE2StorageService.name);
|
| 14 |
+
private s3Client: S3Client;
|
| 15 |
+
private bucket: string;
|
| 16 |
+
private endpoint: string;
|
| 17 |
+
private region: string;
|
| 18 |
+
|
| 19 |
+
constructor(private configService: ConfigService) {
|
| 20 |
+
this.endpoint =
|
| 21 |
+
this.configService.get<string>('idriveE2Storage.endpoint') || '';
|
| 22 |
+
this.region =
|
| 23 |
+
this.configService.get<string>('idriveE2Storage.region') ||
|
| 24 |
+
'ap-southeast-1';
|
| 25 |
+
const accessKeyId =
|
| 26 |
+
this.configService.get<string>('idriveE2Storage.accessKey') || '';
|
| 27 |
+
const secretAccessKey =
|
| 28 |
+
this.configService.get<string>('idriveE2Storage.secretKey') || '';
|
| 29 |
+
this.bucket =
|
| 30 |
+
this.configService.get<string>('idriveE2Storage.bucket') || 'streamflix';
|
| 31 |
+
|
| 32 |
+
this.s3Client = new S3Client({
|
| 33 |
+
region: this.region,
|
| 34 |
+
endpoint: this.endpoint,
|
| 35 |
+
credentials: {
|
| 36 |
+
accessKeyId,
|
| 37 |
+
secretAccessKey,
|
| 38 |
+
},
|
| 39 |
+
forcePathStyle: true,
|
| 40 |
+
});
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
/**
|
| 44 |
+
* Get folder path based on file type
|
| 45 |
+
*/
|
| 46 |
+
private getFolderPath(fileType: string): string {
|
| 47 |
+
const folderMap: Record<string, string> = {
|
| 48 |
+
IMAGE: 'images',
|
| 49 |
+
DOCUMENT: 'documents',
|
| 50 |
+
VIDEO: 'videos',
|
| 51 |
+
};
|
| 52 |
+
return folderMap[fileType.toUpperCase()] || 'others';
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
/**
|
| 56 |
+
* Generate presigned upload URL using S3
|
| 57 |
+
*/
|
| 58 |
+
async getPresignedUploadUrl(fileName: string, fileType?: string) {
|
| 59 |
+
try {
|
| 60 |
+
const folder = fileType ? this.getFolderPath(fileType) : 'others';
|
| 61 |
+
const timestamp = Date.now();
|
| 62 |
+
const filePath = `${folder}/${timestamp}-${fileName}`;
|
| 63 |
+
|
| 64 |
+
const putCommand = new PutObjectCommand({
|
| 65 |
+
Bucket: this.bucket,
|
| 66 |
+
Key: filePath,
|
| 67 |
+
});
|
| 68 |
+
|
| 69 |
+
const uploadUrl = await getSignedUrl(this.s3Client, putCommand, {
|
| 70 |
+
expiresIn: 3600,
|
| 71 |
+
});
|
| 72 |
+
|
| 73 |
+
// For private buckets, generate presigned download URL
|
| 74 |
+
const getCommand = new GetObjectCommand({
|
| 75 |
+
Bucket: this.bucket,
|
| 76 |
+
Key: filePath,
|
| 77 |
+
});
|
| 78 |
+
|
| 79 |
+
const fileUrl = await getSignedUrl(this.s3Client, getCommand, {
|
| 80 |
+
expiresIn: 604800, // 7 days for download URL
|
| 81 |
+
});
|
| 82 |
+
|
| 83 |
+
this.logger.log(
|
| 84 |
+
`Generated presigned upload URL for: ${filePath}${fileType ? ` (type: ${fileType})` : ''}`,
|
| 85 |
+
);
|
| 86 |
+
|
| 87 |
+
return {
|
| 88 |
+
uploadUrl,
|
| 89 |
+
fileUrl, // Presigned download URL (valid for 7 days)
|
| 90 |
+
filePath,
|
| 91 |
+
uploadExpiresIn: 3600, // 1 hour
|
| 92 |
+
downloadExpiresIn: 604800, // 7 days
|
| 93 |
+
...(fileType && { fileType }),
|
| 94 |
+
};
|
| 95 |
+
} catch (error) {
|
| 96 |
+
this.logger.error('Failed to generate presigned upload URL', error);
|
| 97 |
+
throw error;
|
| 98 |
+
}
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
/**
|
| 102 |
+
* Get presigned download URL for an existing file
|
| 103 |
+
*/
|
| 104 |
+
async getPresignedDownloadUrl(filePath: string, expiresIn: number = 604800) {
|
| 105 |
+
try {
|
| 106 |
+
const command = new GetObjectCommand({
|
| 107 |
+
Bucket: this.bucket,
|
| 108 |
+
Key: filePath,
|
| 109 |
+
});
|
| 110 |
+
|
| 111 |
+
const downloadUrl = await getSignedUrl(this.s3Client, command, {
|
| 112 |
+
expiresIn,
|
| 113 |
+
});
|
| 114 |
+
|
| 115 |
+
this.logger.log(`Generated presigned download URL for: ${filePath}`);
|
| 116 |
+
|
| 117 |
+
return {
|
| 118 |
+
downloadUrl,
|
| 119 |
+
filePath,
|
| 120 |
+
expiresIn,
|
| 121 |
+
};
|
| 122 |
+
} catch (error) {
|
| 123 |
+
this.logger.error('Failed to generate presigned download URL', error);
|
| 124 |
+
throw error;
|
| 125 |
+
}
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
/**
|
| 129 |
+
* Delete a file from IDrive e2 Storage
|
| 130 |
+
*/
|
| 131 |
+
async deleteFile(filePath: string): Promise<any> {
|
| 132 |
+
try {
|
| 133 |
+
const command = new DeleteObjectCommand({
|
| 134 |
+
Bucket: this.bucket,
|
| 135 |
+
Key: filePath,
|
| 136 |
+
});
|
| 137 |
+
|
| 138 |
+
const result = await this.s3Client.send(command);
|
| 139 |
+
|
| 140 |
+
this.logger.log(`Successfully deleted file: ${filePath}`);
|
| 141 |
+
return {
|
| 142 |
+
success: true,
|
| 143 |
+
deletedPath: filePath,
|
| 144 |
+
result,
|
| 145 |
+
};
|
| 146 |
+
} catch (error) {
|
| 147 |
+
this.logger.error(`Failed to delete file: ${filePath}`, error);
|
| 148 |
+
throw new NotFoundException('File not found or already deleted');
|
| 149 |
+
}
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
+
/**
|
| 153 |
+
* Delete a file by URL
|
| 154 |
+
*/
|
| 155 |
+
async deleteFileByUrl(url: string): Promise<any> {
|
| 156 |
+
try {
|
| 157 |
+
const filePath = this.extractFilePathFromUrl(url);
|
| 158 |
+
|
| 159 |
+
if (!filePath) {
|
| 160 |
+
throw new NotFoundException('Could not extract file path from URL');
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
const result = await this.deleteFile(filePath);
|
| 164 |
+
|
| 165 |
+
this.logger.log(`Successfully deleted file from URL: ${url}`);
|
| 166 |
+
|
| 167 |
+
return {
|
| 168 |
+
...result,
|
| 169 |
+
deletedUrl: url,
|
| 170 |
+
};
|
| 171 |
+
} catch (error) {
|
| 172 |
+
if (error instanceof NotFoundException) {
|
| 173 |
+
throw error;
|
| 174 |
+
}
|
| 175 |
+
this.logger.error(`Failed to delete file by URL: ${url}`, error);
|
| 176 |
+
throw new NotFoundException('File not found or already deleted');
|
| 177 |
+
}
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
/**
|
| 181 |
+
* Extract file path from IDrive e2 Storage URL
|
| 182 |
+
*/
|
| 183 |
+
private extractFilePathFromUrl(url: string): string | null {
|
| 184 |
+
try {
|
| 185 |
+
// Format: https://streamflix.ap-southeast-1.idrivee2.com/images/123-file.jpg
|
| 186 |
+
const urlObj = new URL(url);
|
| 187 |
+
const path = urlObj.pathname;
|
| 188 |
+
return path.startsWith('/') ? path.substring(1) : path;
|
| 189 |
+
} catch {
|
| 190 |
+
return null;
|
| 191 |
+
}
|
| 192 |
+
}
|
| 193 |
+
}
|
src/shared/modules/upload/upload.module.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { SeaweedfsStorageModule } from './seaweedfs-storage/seaweedfs-storage.mo
|
|
| 9 |
import { FilestackModule } from './filestack/filestack.module';
|
| 10 |
import { GarageStorageModule } from './garage-storage/garage-storage.module';
|
| 11 |
import { RustfsStorageModule } from './rustfs-storage/rustfs-storage.module';
|
|
|
|
| 12 |
|
| 13 |
@Module({
|
| 14 |
imports: [
|
|
@@ -20,6 +21,7 @@ import { RustfsStorageModule } from './rustfs-storage/rustfs-storage.module';
|
|
| 20 |
OldMinioStorageModule,
|
| 21 |
GarageStorageModule,
|
| 22 |
RustfsStorageModule,
|
|
|
|
| 23 |
SeaweedfsStorageModule,
|
| 24 |
FilestackModule,
|
| 25 |
],
|
|
|
|
| 9 |
import { FilestackModule } from './filestack/filestack.module';
|
| 10 |
import { GarageStorageModule } from './garage-storage/garage-storage.module';
|
| 11 |
import { RustfsStorageModule } from './rustfs-storage/rustfs-storage.module';
|
| 12 |
+
import { IdriveE2StorageModule } from './idrive-e2-storage/idrive-e2-storage.module';
|
| 13 |
|
| 14 |
@Module({
|
| 15 |
imports: [
|
|
|
|
| 21 |
OldMinioStorageModule,
|
| 22 |
GarageStorageModule,
|
| 23 |
RustfsStorageModule,
|
| 24 |
+
IdriveE2StorageModule,
|
| 25 |
SeaweedfsStorageModule,
|
| 26 |
FilestackModule,
|
| 27 |
],
|