Akshar2325 commited on
Commit
5d53367
·
1 Parent(s): 51c6e04

feat(idrive-e2-storage): add idrive e2 storage integration

Browse files

- Add IDrive e2 storage module with S3-compatible API integration
- Create file-type enum for categorizing uploads (IMAGE, DOCUMENT, VIDEO)
- Implement presigned URL generation for secure file uploads and downloads
- Add file deletion functionality with URL-based file removal
- Create storage configuration with environment variables for endpoint, region, access key, secret key, and bucket
- Add controller endpoints for presigned upload/download URLs and file deletion
- Implement automatic file organization by type into separate folders
- Add comprehensive error handling and logging throughout the service
- Update environment example file with IDrive e2 credentials configuration
- Integrate IDrive e2 module into main upload module exports

.env.example CHANGED
@@ -103,3 +103,10 @@ RUSTFS_ACCESS_KEY="your-rustfs-access-key"
103
  RUSTFS_SECRET_KEY="your-rustfs-secret-key"
104
  RUSTFS_BUCKET="streamflix"
105
  RUSTFS_REGION="us-east-1"
 
 
 
 
 
 
 
 
103
  RUSTFS_SECRET_KEY="your-rustfs-secret-key"
104
  RUSTFS_BUCKET="streamflix"
105
  RUSTFS_REGION="us-east-1"
106
+
107
+ # IDrive e2 Storage Credentials
108
+ IDRIVE_E2_ENDPOINT="https://s3.ap-southeast-1.idrivee2.com"
109
+ IDRIVE_E2_REGION="ap-southeast-1"
110
+ IDRIVE_E2_ACCESS_KEY="your-idrive-access-key"
111
+ IDRIVE_E2_SECRET_KEY="your-idrive-secret-key"
112
+ IDRIVE_E2_BUCKET="streamflix"
src/shared/modules/upload/idrive-e2-storage/enums/file-type.enum.ts ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ export enum FileType {
2
+ IMAGE = 'IMAGE',
3
+ DOCUMENT = 'DOCUMENT',
4
+ VIDEO = 'VIDEO',
5
+ }
src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.config.ts ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import { registerAs } from '@nestjs/config';
2
+
3
+ export default registerAs('idriveE2Storage', () => ({
4
+ endpoint: process.env.IDRIVE_E2_ENDPOINT,
5
+ region: process.env.IDRIVE_E2_REGION || 'ap-southeast-1',
6
+ accessKey: process.env.IDRIVE_E2_ACCESS_KEY,
7
+ secretKey: process.env.IDRIVE_E2_SECRET_KEY,
8
+ bucket: process.env.IDRIVE_E2_BUCKET || 'streamflix',
9
+ }));
src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.controller.ts ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import {
2
+ Controller,
3
+ Get,
4
+ Delete,
5
+ Query,
6
+ HttpCode,
7
+ HttpStatus,
8
+ } from '@nestjs/common';
9
+ import { ApiTags, ApiOperation, ApiQuery } from '@nestjs/swagger';
10
+ import { IdriveE2StorageService } from './idrive-e2-storage.service';
11
+ import { FileType } from './enums/file-type.enum';
12
+
13
+ @ApiTags('Upload: IDrive e2 Storage')
14
+ @Controller('idrive-e2')
15
+ export class IdriveE2StorageController {
16
+ constructor(
17
+ private readonly idriveE2StorageService: IdriveE2StorageService,
18
+ ) {}
19
+
20
+ @Get('presigned-upload')
21
+ @ApiOperation({ summary: 'Get presigned upload URL' })
22
+ @ApiQuery({ name: 'fileName', required: true })
23
+ @ApiQuery({ name: 'fileType', enum: FileType, required: false })
24
+ async getPresignedUploadUrl(
25
+ @Query('fileName') fileName: string,
26
+ @Query('fileType') fileType?: FileType,
27
+ ) {
28
+ return this.idriveE2StorageService.getPresignedUploadUrl(
29
+ fileName,
30
+ fileType,
31
+ );
32
+ }
33
+
34
+ @Get('presigned-download')
35
+ @ApiOperation({ summary: 'Get presigned download URL for existing file' })
36
+ @ApiQuery({ name: 'filePath', required: true })
37
+ @ApiQuery({
38
+ name: 'expiresIn',
39
+ required: false,
40
+ description: 'Expiration time in seconds (default: 604800)',
41
+ })
42
+ async getPresignedDownloadUrl(
43
+ @Query('filePath') filePath: string,
44
+ @Query('expiresIn') expiresIn?: number,
45
+ ) {
46
+ return this.idriveE2StorageService.getPresignedDownloadUrl(
47
+ filePath,
48
+ expiresIn,
49
+ );
50
+ }
51
+
52
+ @Delete('by-url')
53
+ @HttpCode(HttpStatus.OK)
54
+ @ApiOperation({ summary: 'Delete file by URL' })
55
+ async deleteFileByUrl(@Query('url') url: string) {
56
+ const result = await this.idriveE2StorageService.deleteFileByUrl(url);
57
+ return {
58
+ message: 'File deleted successfully',
59
+ ...result,
60
+ };
61
+ }
62
+ }
src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.module.ts ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { Module } from '@nestjs/common';
2
+ import { ConfigModule } from '@nestjs/config';
3
+ import { IdriveE2StorageController } from './idrive-e2-storage.controller';
4
+ import { IdriveE2StorageService } from './idrive-e2-storage.service';
5
+ import idriveE2StorageConfig from './idrive-e2-storage.config';
6
+
7
+ @Module({
8
+ imports: [ConfigModule.forFeature(idriveE2StorageConfig)],
9
+ controllers: [IdriveE2StorageController],
10
+ providers: [IdriveE2StorageService],
11
+ exports: [IdriveE2StorageService],
12
+ })
13
+ export class IdriveE2StorageModule {}
src/shared/modules/upload/idrive-e2-storage/idrive-e2-storage.service.ts ADDED
@@ -0,0 +1,193 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { Injectable, Logger, NotFoundException } from '@nestjs/common';
2
+ import { ConfigService } from '@nestjs/config';
3
+ import {
4
+ S3Client,
5
+ PutObjectCommand,
6
+ GetObjectCommand,
7
+ DeleteObjectCommand,
8
+ } from '@aws-sdk/client-s3';
9
+ import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
10
+
11
+ @Injectable()
12
+ export class IdriveE2StorageService {
13
+ private readonly logger = new Logger(IdriveE2StorageService.name);
14
+ private s3Client: S3Client;
15
+ private bucket: string;
16
+ private endpoint: string;
17
+ private region: string;
18
+
19
+ constructor(private configService: ConfigService) {
20
+ this.endpoint =
21
+ this.configService.get<string>('idriveE2Storage.endpoint') || '';
22
+ this.region =
23
+ this.configService.get<string>('idriveE2Storage.region') ||
24
+ 'ap-southeast-1';
25
+ const accessKeyId =
26
+ this.configService.get<string>('idriveE2Storage.accessKey') || '';
27
+ const secretAccessKey =
28
+ this.configService.get<string>('idriveE2Storage.secretKey') || '';
29
+ this.bucket =
30
+ this.configService.get<string>('idriveE2Storage.bucket') || 'streamflix';
31
+
32
+ this.s3Client = new S3Client({
33
+ region: this.region,
34
+ endpoint: this.endpoint,
35
+ credentials: {
36
+ accessKeyId,
37
+ secretAccessKey,
38
+ },
39
+ forcePathStyle: true,
40
+ });
41
+ }
42
+
43
+ /**
44
+ * Get folder path based on file type
45
+ */
46
+ private getFolderPath(fileType: string): string {
47
+ const folderMap: Record<string, string> = {
48
+ IMAGE: 'images',
49
+ DOCUMENT: 'documents',
50
+ VIDEO: 'videos',
51
+ };
52
+ return folderMap[fileType.toUpperCase()] || 'others';
53
+ }
54
+
55
+ /**
56
+ * Generate presigned upload URL using S3
57
+ */
58
+ async getPresignedUploadUrl(fileName: string, fileType?: string) {
59
+ try {
60
+ const folder = fileType ? this.getFolderPath(fileType) : 'others';
61
+ const timestamp = Date.now();
62
+ const filePath = `${folder}/${timestamp}-${fileName}`;
63
+
64
+ const putCommand = new PutObjectCommand({
65
+ Bucket: this.bucket,
66
+ Key: filePath,
67
+ });
68
+
69
+ const uploadUrl = await getSignedUrl(this.s3Client, putCommand, {
70
+ expiresIn: 3600,
71
+ });
72
+
73
+ // For private buckets, generate presigned download URL
74
+ const getCommand = new GetObjectCommand({
75
+ Bucket: this.bucket,
76
+ Key: filePath,
77
+ });
78
+
79
+ const fileUrl = await getSignedUrl(this.s3Client, getCommand, {
80
+ expiresIn: 604800, // 7 days for download URL
81
+ });
82
+
83
+ this.logger.log(
84
+ `Generated presigned upload URL for: ${filePath}${fileType ? ` (type: ${fileType})` : ''}`,
85
+ );
86
+
87
+ return {
88
+ uploadUrl,
89
+ fileUrl, // Presigned download URL (valid for 7 days)
90
+ filePath,
91
+ uploadExpiresIn: 3600, // 1 hour
92
+ downloadExpiresIn: 604800, // 7 days
93
+ ...(fileType && { fileType }),
94
+ };
95
+ } catch (error) {
96
+ this.logger.error('Failed to generate presigned upload URL', error);
97
+ throw error;
98
+ }
99
+ }
100
+
101
+ /**
102
+ * Get presigned download URL for an existing file
103
+ */
104
+ async getPresignedDownloadUrl(filePath: string, expiresIn: number = 604800) {
105
+ try {
106
+ const command = new GetObjectCommand({
107
+ Bucket: this.bucket,
108
+ Key: filePath,
109
+ });
110
+
111
+ const downloadUrl = await getSignedUrl(this.s3Client, command, {
112
+ expiresIn,
113
+ });
114
+
115
+ this.logger.log(`Generated presigned download URL for: ${filePath}`);
116
+
117
+ return {
118
+ downloadUrl,
119
+ filePath,
120
+ expiresIn,
121
+ };
122
+ } catch (error) {
123
+ this.logger.error('Failed to generate presigned download URL', error);
124
+ throw error;
125
+ }
126
+ }
127
+
128
+ /**
129
+ * Delete a file from IDrive e2 Storage
130
+ */
131
+ async deleteFile(filePath: string): Promise<any> {
132
+ try {
133
+ const command = new DeleteObjectCommand({
134
+ Bucket: this.bucket,
135
+ Key: filePath,
136
+ });
137
+
138
+ const result = await this.s3Client.send(command);
139
+
140
+ this.logger.log(`Successfully deleted file: ${filePath}`);
141
+ return {
142
+ success: true,
143
+ deletedPath: filePath,
144
+ result,
145
+ };
146
+ } catch (error) {
147
+ this.logger.error(`Failed to delete file: ${filePath}`, error);
148
+ throw new NotFoundException('File not found or already deleted');
149
+ }
150
+ }
151
+
152
+ /**
153
+ * Delete a file by URL
154
+ */
155
+ async deleteFileByUrl(url: string): Promise<any> {
156
+ try {
157
+ const filePath = this.extractFilePathFromUrl(url);
158
+
159
+ if (!filePath) {
160
+ throw new NotFoundException('Could not extract file path from URL');
161
+ }
162
+
163
+ const result = await this.deleteFile(filePath);
164
+
165
+ this.logger.log(`Successfully deleted file from URL: ${url}`);
166
+
167
+ return {
168
+ ...result,
169
+ deletedUrl: url,
170
+ };
171
+ } catch (error) {
172
+ if (error instanceof NotFoundException) {
173
+ throw error;
174
+ }
175
+ this.logger.error(`Failed to delete file by URL: ${url}`, error);
176
+ throw new NotFoundException('File not found or already deleted');
177
+ }
178
+ }
179
+
180
+ /**
181
+ * Extract file path from IDrive e2 Storage URL
182
+ */
183
+ private extractFilePathFromUrl(url: string): string | null {
184
+ try {
185
+ // Format: https://streamflix.ap-southeast-1.idrivee2.com/images/123-file.jpg
186
+ const urlObj = new URL(url);
187
+ const path = urlObj.pathname;
188
+ return path.startsWith('/') ? path.substring(1) : path;
189
+ } catch {
190
+ return null;
191
+ }
192
+ }
193
+ }
src/shared/modules/upload/upload.module.ts CHANGED
@@ -9,6 +9,7 @@ import { SeaweedfsStorageModule } from './seaweedfs-storage/seaweedfs-storage.mo
9
  import { FilestackModule } from './filestack/filestack.module';
10
  import { GarageStorageModule } from './garage-storage/garage-storage.module';
11
  import { RustfsStorageModule } from './rustfs-storage/rustfs-storage.module';
 
12
 
13
  @Module({
14
  imports: [
@@ -20,6 +21,7 @@ import { RustfsStorageModule } from './rustfs-storage/rustfs-storage.module';
20
  OldMinioStorageModule,
21
  GarageStorageModule,
22
  RustfsStorageModule,
 
23
  SeaweedfsStorageModule,
24
  FilestackModule,
25
  ],
 
9
  import { FilestackModule } from './filestack/filestack.module';
10
  import { GarageStorageModule } from './garage-storage/garage-storage.module';
11
  import { RustfsStorageModule } from './rustfs-storage/rustfs-storage.module';
12
+ import { IdriveE2StorageModule } from './idrive-e2-storage/idrive-e2-storage.module';
13
 
14
  @Module({
15
  imports: [
 
21
  OldMinioStorageModule,
22
  GarageStorageModule,
23
  RustfsStorageModule,
24
+ IdriveE2StorageModule,
25
  SeaweedfsStorageModule,
26
  FilestackModule,
27
  ],