relv-dev commited on
Commit
e518a85
·
verified ·
1 Parent(s): 2db5489

Add R2 dual-storage adapter and resilient legacy large-document wait

Browse files
.env.example CHANGED
@@ -19,11 +19,18 @@ ASSIGNMENT_TITLE=HTLLP notes
19
  # Playwright
20
  PLAYWRIGHT_HEADLESS=true
21
 
22
- # Storage buckets
23
- INPUT_BUCKET=turnitin-inputs
24
- REPORT_BUCKET=turnitin-reports
25
- DIAGNOSTICS_BUCKET=turnitin-diagnostics
26
- SESSION_BUCKET=turnitin-sessions
 
 
 
 
 
 
 
27
 
28
  # Cron intervals (minutes)
29
  QUOTA_CHECK_INTERVAL=30
 
19
  # Playwright
20
  PLAYWRIGHT_HEADLESS=true
21
 
22
+ # Object storage. Existing plain paths continue to read from Supabase after
23
+ # switching new writes to R2.
24
+ STORAGE_PROVIDER=supabase
25
+ INPUT_BUCKET=relv-turnitin-inputs
26
+ REPORT_BUCKET=relv-turnitin-reports
27
+ DIAGNOSTICS_BUCKET=relv-turnitin-diagnostics
28
+ SESSION_BUCKET=relv-turnitin-sessions
29
+ R2_ACCOUNT_ID=
30
+ R2_ACCESS_KEY_ID=
31
+ R2_SECRET_ACCESS_KEY=
32
+ R2_ENDPOINT=https://ACCOUNT_ID.r2.cloudflarestorage.com
33
+ R2_REGION=auto
34
 
35
  # Cron intervals (minutes)
36
  QUOTA_CHECK_INTERVAL=30
package-lock.json CHANGED
@@ -8,14 +8,19 @@
8
  "name": "turnitin-worker",
9
  "version": "1.0.0",
10
  "dependencies": {
 
 
11
  "@supabase/supabase-js": "^2.49.0",
 
12
  "dotenv": "^16.4.5",
13
  "express": "^4.21.0",
14
  "multer": "^1.4.5-lts.1",
15
  "node-cron": "^3.0.3",
 
16
  "playwright": "^1.49.0"
17
  },
18
  "devDependencies": {
 
19
  "@types/express": "^5.0.0",
20
  "@types/multer": "^1.4.12",
21
  "@types/node": "^22.0.0",
@@ -27,6 +32,331 @@
27
  "node": ">=20"
28
  }
29
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  "node_modules/@esbuild/aix-ppc64": {
31
  "version": "0.28.1",
32
  "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz",
@@ -469,6 +799,105 @@
469
  "node": ">=18"
470
  }
471
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
472
  "node_modules/@supabase/auth-js": {
473
  "version": "2.110.0",
474
  "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.110.0.tgz",
@@ -553,6 +982,16 @@
553
  "node": ">=22.0.0"
554
  }
555
  },
 
 
 
 
 
 
 
 
 
 
556
  "node_modules/@types/body-parser": {
557
  "version": "1.19.6",
558
  "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz",
@@ -681,6 +1120,15 @@
681
  "node": ">= 0.6"
682
  }
683
  },
 
 
 
 
 
 
 
 
 
684
  "node_modules/append-field": {
685
  "version": "1.0.0",
686
  "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
@@ -717,6 +1165,12 @@
717
  "npm": "1.2.8000 || >= 1.4.16"
718
  }
719
  },
 
 
 
 
 
 
720
  "node_modules/buffer-from": {
721
  "version": "1.1.2",
722
  "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
@@ -1394,6 +1848,12 @@
1394
  "node": ">= 0.8"
1395
  }
1396
  },
 
 
 
 
 
 
1397
  "node_modules/parseurl": {
1398
  "version": "1.3.3",
1399
  "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -1409,6 +1869,24 @@
1409
  "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
1410
  "license": "MIT"
1411
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1412
  "node_modules/playwright": {
1413
  "version": "1.61.1",
1414
  "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz",
 
8
  "name": "turnitin-worker",
9
  "version": "1.0.0",
10
  "dependencies": {
11
+ "@aws-sdk/client-s3": "^3.1089.0",
12
+ "@aws-sdk/s3-request-presigner": "^3.1089.0",
13
  "@supabase/supabase-js": "^2.49.0",
14
+ "adm-zip": "^0.6.0",
15
  "dotenv": "^16.4.5",
16
  "express": "^4.21.0",
17
  "multer": "^1.4.5-lts.1",
18
  "node-cron": "^3.0.3",
19
+ "pdf-lib": "^1.17.1",
20
  "playwright": "^1.49.0"
21
  },
22
  "devDependencies": {
23
+ "@types/adm-zip": "^0.5.8",
24
  "@types/express": "^5.0.0",
25
  "@types/multer": "^1.4.12",
26
  "@types/node": "^22.0.0",
 
32
  "node": ">=20"
33
  }
34
  },
35
+ "node_modules/@aws-sdk/checksums": {
36
+ "version": "3.1000.18",
37
+ "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.18.tgz",
38
+ "integrity": "sha512-IImkbEyXdV6/uaF5r6Wkk+8718mQw1ll83j0a4a30R3JM/rHVFdWAiT4jtJpFjJiIwM/oJ6SxIxr0z2TaQUGqw==",
39
+ "license": "Apache-2.0",
40
+ "dependencies": {
41
+ "@aws-sdk/core": "^3.975.3",
42
+ "@aws-sdk/types": "^3.974.2",
43
+ "@smithy/core": "^3.29.4",
44
+ "@smithy/types": "^4.16.1",
45
+ "tslib": "^2.6.2"
46
+ },
47
+ "engines": {
48
+ "node": ">=20.0.0"
49
+ }
50
+ },
51
+ "node_modules/@aws-sdk/client-s3": {
52
+ "version": "3.1089.0",
53
+ "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1089.0.tgz",
54
+ "integrity": "sha512-Sc+JOtNeP+v+XdP9Rb4Hh+uhiNO2hh/CZQbKYooNakhOIgK6/K0cjoDCEGeFmkG0vf2DopTmSctzKlcKwy1BPw==",
55
+ "license": "Apache-2.0",
56
+ "dependencies": {
57
+ "@aws-sdk/checksums": "^3.1000.18",
58
+ "@aws-sdk/core": "^3.975.3",
59
+ "@aws-sdk/credential-provider-node": "^3.972.70",
60
+ "@aws-sdk/middleware-sdk-s3": "^3.972.64",
61
+ "@aws-sdk/signature-v4-multi-region": "^3.996.41",
62
+ "@aws-sdk/types": "^3.974.2",
63
+ "@smithy/core": "^3.29.4",
64
+ "@smithy/fetch-http-handler": "^5.6.6",
65
+ "@smithy/node-http-handler": "^4.9.6",
66
+ "@smithy/types": "^4.16.1",
67
+ "tslib": "^2.6.2"
68
+ },
69
+ "engines": {
70
+ "node": ">=20.0.0"
71
+ }
72
+ },
73
+ "node_modules/@aws-sdk/core": {
74
+ "version": "3.975.3",
75
+ "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.975.3.tgz",
76
+ "integrity": "sha512-7ur3kCKuvPLqlsZ2XlvnNBVQ7KkpSu6Y6dOTwSPHLrFpTEfZM8isLBJc4cgv96WB7GifeVM436mpycwxBd2vEA==",
77
+ "license": "Apache-2.0",
78
+ "dependencies": {
79
+ "@aws-sdk/types": "^3.974.2",
80
+ "@aws-sdk/xml-builder": "^3.972.36",
81
+ "@aws/lambda-invoke-store": "^0.3.0",
82
+ "@smithy/core": "^3.29.4",
83
+ "@smithy/signature-v4": "^5.6.5",
84
+ "@smithy/types": "^4.16.1",
85
+ "bowser": "^2.11.0",
86
+ "tslib": "^2.6.2"
87
+ },
88
+ "engines": {
89
+ "node": ">=20.0.0"
90
+ }
91
+ },
92
+ "node_modules/@aws-sdk/credential-provider-env": {
93
+ "version": "3.972.59",
94
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.59.tgz",
95
+ "integrity": "sha512-Ny5e4Mfh3QPmiAc0AiUe+cbTXDlxkU3Rc+EpWOfyWeWEy6yp7Fa1KmfNeCc+1a8by9zQ9gtohmiQUkMPScF3ng==",
96
+ "license": "Apache-2.0",
97
+ "dependencies": {
98
+ "@aws-sdk/core": "^3.975.3",
99
+ "@aws-sdk/types": "^3.974.2",
100
+ "@smithy/core": "^3.29.4",
101
+ "@smithy/types": "^4.16.1",
102
+ "tslib": "^2.6.2"
103
+ },
104
+ "engines": {
105
+ "node": ">=20.0.0"
106
+ }
107
+ },
108
+ "node_modules/@aws-sdk/credential-provider-http": {
109
+ "version": "3.972.61",
110
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.61.tgz",
111
+ "integrity": "sha512-8jAjgStl5Ytq4+HF3X/9f+EmRinaRbGRRtQGktlPfBRVx73H+R1y48vIeXerQtYGFaUqkEp3fT6jP854rVO2yQ==",
112
+ "license": "Apache-2.0",
113
+ "dependencies": {
114
+ "@aws-sdk/core": "^3.975.3",
115
+ "@aws-sdk/types": "^3.974.2",
116
+ "@smithy/core": "^3.29.4",
117
+ "@smithy/fetch-http-handler": "^5.6.6",
118
+ "@smithy/node-http-handler": "^4.9.6",
119
+ "@smithy/types": "^4.16.1",
120
+ "tslib": "^2.6.2"
121
+ },
122
+ "engines": {
123
+ "node": ">=20.0.0"
124
+ }
125
+ },
126
+ "node_modules/@aws-sdk/credential-provider-ini": {
127
+ "version": "3.973.4",
128
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.4.tgz",
129
+ "integrity": "sha512-e6ZvVsj90aRALf1kHP+J4iqC1496ZpVgqI/+u0LJ5HL7q7ATauGy4gdDvRCP13L1pN/fMiZLah162PGIYkbUVQ==",
130
+ "license": "Apache-2.0",
131
+ "dependencies": {
132
+ "@aws-sdk/core": "^3.975.3",
133
+ "@aws-sdk/credential-provider-env": "^3.972.59",
134
+ "@aws-sdk/credential-provider-http": "^3.972.61",
135
+ "@aws-sdk/credential-provider-login": "^3.972.66",
136
+ "@aws-sdk/credential-provider-process": "^3.972.59",
137
+ "@aws-sdk/credential-provider-sso": "^3.973.3",
138
+ "@aws-sdk/credential-provider-web-identity": "^3.972.65",
139
+ "@aws-sdk/nested-clients": "^3.997.33",
140
+ "@aws-sdk/types": "^3.974.2",
141
+ "@smithy/core": "^3.29.4",
142
+ "@smithy/credential-provider-imds": "^4.4.9",
143
+ "@smithy/types": "^4.16.1",
144
+ "tslib": "^2.6.2"
145
+ },
146
+ "engines": {
147
+ "node": ">=20.0.0"
148
+ }
149
+ },
150
+ "node_modules/@aws-sdk/credential-provider-login": {
151
+ "version": "3.972.66",
152
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.66.tgz",
153
+ "integrity": "sha512-g2fsqm87r/nKthLZ0VkkDBElkGg0PvSa8d97HQ6EilMbJTZ6hxa8FxkSZyJfgPfFdZn0TTmkOffQmTSUcAHIng==",
154
+ "license": "Apache-2.0",
155
+ "dependencies": {
156
+ "@aws-sdk/core": "^3.975.3",
157
+ "@aws-sdk/nested-clients": "^3.997.33",
158
+ "@aws-sdk/types": "^3.974.2",
159
+ "@smithy/core": "^3.29.4",
160
+ "@smithy/types": "^4.16.1",
161
+ "tslib": "^2.6.2"
162
+ },
163
+ "engines": {
164
+ "node": ">=20.0.0"
165
+ }
166
+ },
167
+ "node_modules/@aws-sdk/credential-provider-node": {
168
+ "version": "3.972.70",
169
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.70.tgz",
170
+ "integrity": "sha512-3xzvkGdykBunxqh8WudmUpSyLWvIhfI6aBQo1b5rb3mDO5mNLadK+0hiI0qBQBMVynJbfLO+Ajy9dztMwy9O8w==",
171
+ "license": "Apache-2.0",
172
+ "dependencies": {
173
+ "@aws-sdk/credential-provider-env": "^3.972.59",
174
+ "@aws-sdk/credential-provider-http": "^3.972.61",
175
+ "@aws-sdk/credential-provider-ini": "^3.973.4",
176
+ "@aws-sdk/credential-provider-process": "^3.972.59",
177
+ "@aws-sdk/credential-provider-sso": "^3.973.3",
178
+ "@aws-sdk/credential-provider-web-identity": "^3.972.65",
179
+ "@aws-sdk/types": "^3.974.2",
180
+ "@smithy/core": "^3.29.4",
181
+ "@smithy/credential-provider-imds": "^4.4.9",
182
+ "@smithy/types": "^4.16.1",
183
+ "tslib": "^2.6.2"
184
+ },
185
+ "engines": {
186
+ "node": ">=20.0.0"
187
+ }
188
+ },
189
+ "node_modules/@aws-sdk/credential-provider-process": {
190
+ "version": "3.972.59",
191
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.59.tgz",
192
+ "integrity": "sha512-DlZF2/MhLlatDdlrIy3CUCpfdbLrKx+3SMjVo+WyHnPpwzkc/M3vwAHw4OVJf7DMvO+4vfRqSCMc/E9I1auN0g==",
193
+ "license": "Apache-2.0",
194
+ "dependencies": {
195
+ "@aws-sdk/core": "^3.975.3",
196
+ "@aws-sdk/types": "^3.974.2",
197
+ "@smithy/core": "^3.29.4",
198
+ "@smithy/types": "^4.16.1",
199
+ "tslib": "^2.6.2"
200
+ },
201
+ "engines": {
202
+ "node": ">=20.0.0"
203
+ }
204
+ },
205
+ "node_modules/@aws-sdk/credential-provider-sso": {
206
+ "version": "3.973.3",
207
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.3.tgz",
208
+ "integrity": "sha512-hmdDHoy2G5Es2e8IgelNMYUuSQI6uCIAKZMJ2u2PdKDhxvbk1uWD/g4+R7R5c/tJfKEB1+KjjWiaoCr/S+ZTiQ==",
209
+ "license": "Apache-2.0",
210
+ "dependencies": {
211
+ "@aws-sdk/core": "^3.975.3",
212
+ "@aws-sdk/nested-clients": "^3.997.33",
213
+ "@aws-sdk/token-providers": "3.1088.0",
214
+ "@aws-sdk/types": "^3.974.2",
215
+ "@smithy/core": "^3.29.4",
216
+ "@smithy/types": "^4.16.1",
217
+ "tslib": "^2.6.2"
218
+ },
219
+ "engines": {
220
+ "node": ">=20.0.0"
221
+ }
222
+ },
223
+ "node_modules/@aws-sdk/credential-provider-web-identity": {
224
+ "version": "3.972.65",
225
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.65.tgz",
226
+ "integrity": "sha512-gHQb/Kt0chjk/JQDa/GJDqmAvEuVn8n7z10wK2h0LFM9TUDRkohgOO4aEF+s2sBLM0br7Cl5W6P7phgjrrJvLQ==",
227
+ "license": "Apache-2.0",
228
+ "dependencies": {
229
+ "@aws-sdk/core": "^3.975.3",
230
+ "@aws-sdk/nested-clients": "^3.997.33",
231
+ "@aws-sdk/types": "^3.974.2",
232
+ "@smithy/core": "^3.29.4",
233
+ "@smithy/types": "^4.16.1",
234
+ "tslib": "^2.6.2"
235
+ },
236
+ "engines": {
237
+ "node": ">=20.0.0"
238
+ }
239
+ },
240
+ "node_modules/@aws-sdk/middleware-sdk-s3": {
241
+ "version": "3.972.64",
242
+ "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.64.tgz",
243
+ "integrity": "sha512-RBi43anhDBUv+HCfxCOXwGOE7GmT4n7ChV04Mwr22RhXTNcamW/iWnJlOotDPCZSrJ4dEvhZSiWWQMwLX+ZhFA==",
244
+ "license": "Apache-2.0",
245
+ "dependencies": {
246
+ "@aws-sdk/core": "^3.975.3",
247
+ "@aws-sdk/signature-v4-multi-region": "^3.996.41",
248
+ "@aws-sdk/types": "^3.974.2",
249
+ "@smithy/core": "^3.29.4",
250
+ "@smithy/types": "^4.16.1",
251
+ "tslib": "^2.6.2"
252
+ },
253
+ "engines": {
254
+ "node": ">=20.0.0"
255
+ }
256
+ },
257
+ "node_modules/@aws-sdk/nested-clients": {
258
+ "version": "3.997.33",
259
+ "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.33.tgz",
260
+ "integrity": "sha512-dVZOroI/r3/ENvqNGgjMPul+jjlz9GddfVusgTXlVjfZj5isibOxecLkGQbRPp8XOuX+RAfjXLFgPkD1JS5xrw==",
261
+ "license": "Apache-2.0",
262
+ "dependencies": {
263
+ "@aws-sdk/core": "^3.975.3",
264
+ "@aws-sdk/signature-v4-multi-region": "^3.996.41",
265
+ "@aws-sdk/types": "^3.974.2",
266
+ "@smithy/core": "^3.29.4",
267
+ "@smithy/fetch-http-handler": "^5.6.6",
268
+ "@smithy/node-http-handler": "^4.9.6",
269
+ "@smithy/types": "^4.16.1",
270
+ "tslib": "^2.6.2"
271
+ },
272
+ "engines": {
273
+ "node": ">=20.0.0"
274
+ }
275
+ },
276
+ "node_modules/@aws-sdk/s3-request-presigner": {
277
+ "version": "3.1089.0",
278
+ "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.1089.0.tgz",
279
+ "integrity": "sha512-NYLT340eRdqv8XIUEIEWcIYICDxBGxpgdyQ9veBkW5XxlRDNn19IZ8b46ddec+HBQea/18W+/IfogCqbac3CGQ==",
280
+ "license": "Apache-2.0",
281
+ "dependencies": {
282
+ "@aws-sdk/core": "^3.975.3",
283
+ "@aws-sdk/signature-v4-multi-region": "^3.996.41",
284
+ "@aws-sdk/types": "^3.974.2",
285
+ "@smithy/core": "^3.29.4",
286
+ "@smithy/types": "^4.16.1",
287
+ "tslib": "^2.6.2"
288
+ },
289
+ "engines": {
290
+ "node": ">=20.0.0"
291
+ }
292
+ },
293
+ "node_modules/@aws-sdk/signature-v4-multi-region": {
294
+ "version": "3.996.41",
295
+ "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.41.tgz",
296
+ "integrity": "sha512-QMUytg+FQMGouc8gHS00KoYih3+N6cqmVI/pQGOIo7Nr7OpQaiXjSYOuL+vsPZ1tymY4LAQ8MYcHJmws5LRxng==",
297
+ "license": "Apache-2.0",
298
+ "dependencies": {
299
+ "@aws-sdk/types": "^3.974.2",
300
+ "@smithy/signature-v4": "^5.6.5",
301
+ "@smithy/types": "^4.16.1",
302
+ "tslib": "^2.6.2"
303
+ },
304
+ "engines": {
305
+ "node": ">=20.0.0"
306
+ }
307
+ },
308
+ "node_modules/@aws-sdk/token-providers": {
309
+ "version": "3.1088.0",
310
+ "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1088.0.tgz",
311
+ "integrity": "sha512-4ObatWt2qpJg5FBk4LOOKrTQYzaqeewAtdO3r9ZO8lH9YqLtpTzLyIdy0mJ+nVdfYOnqISkKNfmzP22bNDhwyw==",
312
+ "license": "Apache-2.0",
313
+ "dependencies": {
314
+ "@aws-sdk/core": "^3.975.3",
315
+ "@aws-sdk/nested-clients": "^3.997.33",
316
+ "@aws-sdk/types": "^3.974.2",
317
+ "@smithy/core": "^3.29.4",
318
+ "@smithy/types": "^4.16.1",
319
+ "tslib": "^2.6.2"
320
+ },
321
+ "engines": {
322
+ "node": ">=20.0.0"
323
+ }
324
+ },
325
+ "node_modules/@aws-sdk/types": {
326
+ "version": "3.974.2",
327
+ "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.2.tgz",
328
+ "integrity": "sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==",
329
+ "license": "Apache-2.0",
330
+ "dependencies": {
331
+ "@smithy/types": "^4.16.1",
332
+ "tslib": "^2.6.2"
333
+ },
334
+ "engines": {
335
+ "node": ">=20.0.0"
336
+ }
337
+ },
338
+ "node_modules/@aws-sdk/xml-builder": {
339
+ "version": "3.972.36",
340
+ "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.36.tgz",
341
+ "integrity": "sha512-RdGmS1GLrtaTOLE1ElSluMldNrpk9Emq6uYs8SS8iHlu5xTAmM9rRkM91o48+rIRryBtyO9t+uLYCoMG6jVMVA==",
342
+ "license": "Apache-2.0",
343
+ "dependencies": {
344
+ "@smithy/types": "^4.16.1",
345
+ "tslib": "^2.6.2"
346
+ },
347
+ "engines": {
348
+ "node": ">=20.0.0"
349
+ }
350
+ },
351
+ "node_modules/@aws/lambda-invoke-store": {
352
+ "version": "0.3.0",
353
+ "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz",
354
+ "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==",
355
+ "license": "Apache-2.0",
356
+ "engines": {
357
+ "node": ">=18.0.0"
358
+ }
359
+ },
360
  "node_modules/@esbuild/aix-ppc64": {
361
  "version": "0.28.1",
362
  "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz",
 
799
  "node": ">=18"
800
  }
801
  },
802
+ "node_modules/@pdf-lib/standard-fonts": {
803
+ "version": "1.0.0",
804
+ "resolved": "https://registry.npmjs.org/@pdf-lib/standard-fonts/-/standard-fonts-1.0.0.tgz",
805
+ "integrity": "sha512-hU30BK9IUN/su0Mn9VdlVKsWBS6GyhVfqjwl1FjZN4TxP6cCw0jP2w7V3Hf5uX7M0AZJ16vey9yE0ny7Sa59ZA==",
806
+ "license": "MIT",
807
+ "dependencies": {
808
+ "pako": "^1.0.6"
809
+ }
810
+ },
811
+ "node_modules/@pdf-lib/upng": {
812
+ "version": "1.0.1",
813
+ "resolved": "https://registry.npmjs.org/@pdf-lib/upng/-/upng-1.0.1.tgz",
814
+ "integrity": "sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==",
815
+ "license": "MIT",
816
+ "dependencies": {
817
+ "pako": "^1.0.10"
818
+ }
819
+ },
820
+ "node_modules/@smithy/core": {
821
+ "version": "3.29.5",
822
+ "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.5.tgz",
823
+ "integrity": "sha512-i0dk2t5B+CwV/dcJdUHILYkOQF5lof8f44dFCfDWToGCxjT9YQ+CgHqTAvJxzc3+zqQwm2QtVoJ5IqiNar/CnQ==",
824
+ "license": "Apache-2.0",
825
+ "dependencies": {
826
+ "@smithy/types": "^4.16.1",
827
+ "tslib": "^2.6.2"
828
+ },
829
+ "engines": {
830
+ "node": ">=18.0.0"
831
+ }
832
+ },
833
+ "node_modules/@smithy/credential-provider-imds": {
834
+ "version": "4.4.10",
835
+ "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.10.tgz",
836
+ "integrity": "sha512-MJenAe4OKRZUo1LdYYFDCsSHxaHvInIU/z52GsheO9vl1/VSySVCr0zkyKD6TFiGkSUaWGxvKZ/70OvgUZR5HQ==",
837
+ "license": "Apache-2.0",
838
+ "dependencies": {
839
+ "@smithy/core": "^3.29.5",
840
+ "@smithy/types": "^4.16.1",
841
+ "tslib": "^2.6.2"
842
+ },
843
+ "engines": {
844
+ "node": ">=18.0.0"
845
+ }
846
+ },
847
+ "node_modules/@smithy/fetch-http-handler": {
848
+ "version": "5.6.7",
849
+ "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.7.tgz",
850
+ "integrity": "sha512-3zpg8yqqyXzoK2TsRDdkqVOj2RDBFfLXwCczOZ5c7TWB4eiaebfSCsbMjDPYB3PJ9ihV62QaeadZ+wLadZtNGA==",
851
+ "license": "Apache-2.0",
852
+ "dependencies": {
853
+ "@smithy/core": "^3.29.5",
854
+ "@smithy/types": "^4.16.1",
855
+ "tslib": "^2.6.2"
856
+ },
857
+ "engines": {
858
+ "node": ">=18.0.0"
859
+ }
860
+ },
861
+ "node_modules/@smithy/node-http-handler": {
862
+ "version": "4.9.7",
863
+ "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.7.tgz",
864
+ "integrity": "sha512-wCU8HCLjAtAVqxxe0j2xff9LcEPw3yjBbg5IdQDIYFnxnPxbxcSLc7rgex7kqm9L/WYOnJEgaWQlfDkZleozMA==",
865
+ "license": "Apache-2.0",
866
+ "dependencies": {
867
+ "@smithy/core": "^3.29.5",
868
+ "@smithy/types": "^4.16.1",
869
+ "tslib": "^2.6.2"
870
+ },
871
+ "engines": {
872
+ "node": ">=18.0.0"
873
+ }
874
+ },
875
+ "node_modules/@smithy/signature-v4": {
876
+ "version": "5.6.6",
877
+ "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.6.tgz",
878
+ "integrity": "sha512-efP6DN3UTFrzIsGO42/xcabv8jU7+9nwEdphFUH7yL0k010ERyAWaO41KFQIDLcFZLZ8xzIQr4wplFxNzslSGQ==",
879
+ "license": "Apache-2.0",
880
+ "dependencies": {
881
+ "@smithy/core": "^3.29.5",
882
+ "@smithy/types": "^4.16.1",
883
+ "tslib": "^2.6.2"
884
+ },
885
+ "engines": {
886
+ "node": ">=18.0.0"
887
+ }
888
+ },
889
+ "node_modules/@smithy/types": {
890
+ "version": "4.16.1",
891
+ "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz",
892
+ "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==",
893
+ "license": "Apache-2.0",
894
+ "dependencies": {
895
+ "tslib": "^2.6.2"
896
+ },
897
+ "engines": {
898
+ "node": ">=18.0.0"
899
+ }
900
+ },
901
  "node_modules/@supabase/auth-js": {
902
  "version": "2.110.0",
903
  "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.110.0.tgz",
 
982
  "node": ">=22.0.0"
983
  }
984
  },
985
+ "node_modules/@types/adm-zip": {
986
+ "version": "0.5.8",
987
+ "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.8.tgz",
988
+ "integrity": "sha512-RVVH7QvZYbN+ihqZ4kX/dMiowf6o+Jk1fNwiSdx0NahBJLU787zkULhGhJM8mf/obmLGmgdMM0bXsQTmyfbR7Q==",
989
+ "dev": true,
990
+ "license": "MIT",
991
+ "dependencies": {
992
+ "@types/node": "*"
993
+ }
994
+ },
995
  "node_modules/@types/body-parser": {
996
  "version": "1.19.6",
997
  "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz",
 
1120
  "node": ">= 0.6"
1121
  }
1122
  },
1123
+ "node_modules/adm-zip": {
1124
+ "version": "0.6.0",
1125
+ "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.0.tgz",
1126
+ "integrity": "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg==",
1127
+ "license": "MIT",
1128
+ "engines": {
1129
+ "node": ">=14.0"
1130
+ }
1131
+ },
1132
  "node_modules/append-field": {
1133
  "version": "1.0.0",
1134
  "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
 
1165
  "npm": "1.2.8000 || >= 1.4.16"
1166
  }
1167
  },
1168
+ "node_modules/bowser": {
1169
+ "version": "2.14.1",
1170
+ "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz",
1171
+ "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==",
1172
+ "license": "MIT"
1173
+ },
1174
  "node_modules/buffer-from": {
1175
  "version": "1.1.2",
1176
  "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
 
1848
  "node": ">= 0.8"
1849
  }
1850
  },
1851
+ "node_modules/pako": {
1852
+ "version": "1.0.11",
1853
+ "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
1854
+ "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==",
1855
+ "license": "(MIT AND Zlib)"
1856
+ },
1857
  "node_modules/parseurl": {
1858
  "version": "1.3.3",
1859
  "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
 
1869
  "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
1870
  "license": "MIT"
1871
  },
1872
+ "node_modules/pdf-lib": {
1873
+ "version": "1.17.1",
1874
+ "resolved": "https://registry.npmjs.org/pdf-lib/-/pdf-lib-1.17.1.tgz",
1875
+ "integrity": "sha512-V/mpyJAoTsN4cnP31vc0wfNA1+p20evqqnap0KLoRUN0Yk/p3wN52DOEsL4oBFcLdb76hlpKPtzJIgo67j/XLw==",
1876
+ "license": "MIT",
1877
+ "dependencies": {
1878
+ "@pdf-lib/standard-fonts": "^1.0.0",
1879
+ "@pdf-lib/upng": "^1.0.1",
1880
+ "pako": "^1.0.11",
1881
+ "tslib": "^1.11.1"
1882
+ }
1883
+ },
1884
+ "node_modules/pdf-lib/node_modules/tslib": {
1885
+ "version": "1.14.1",
1886
+ "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
1887
+ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
1888
+ "license": "0BSD"
1889
+ },
1890
  "node_modules/playwright": {
1891
  "version": "1.61.1",
1892
  "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz",
package.json CHANGED
@@ -10,14 +10,19 @@
10
  "typecheck": "tsc --noEmit"
11
  },
12
  "dependencies": {
 
 
13
  "@supabase/supabase-js": "^2.49.0",
 
14
  "dotenv": "^16.4.5",
15
  "express": "^4.21.0",
16
  "multer": "^1.4.5-lts.1",
17
  "node-cron": "^3.0.3",
 
18
  "playwright": "^1.49.0"
19
  },
20
  "devDependencies": {
 
21
  "@types/express": "^5.0.0",
22
  "@types/multer": "^1.4.12",
23
  "@types/node": "^22.0.0",
 
10
  "typecheck": "tsc --noEmit"
11
  },
12
  "dependencies": {
13
+ "@aws-sdk/client-s3": "^3.1089.0",
14
+ "@aws-sdk/s3-request-presigner": "^3.1089.0",
15
  "@supabase/supabase-js": "^2.49.0",
16
+ "adm-zip": "^0.6.0",
17
  "dotenv": "^16.4.5",
18
  "express": "^4.21.0",
19
  "multer": "^1.4.5-lts.1",
20
  "node-cron": "^3.0.3",
21
+ "pdf-lib": "^1.17.1",
22
  "playwright": "^1.49.0"
23
  },
24
  "devDependencies": {
25
+ "@types/adm-zip": "^0.5.8",
26
  "@types/express": "^5.0.0",
27
  "@types/multer": "^1.4.12",
28
  "@types/node": "^22.0.0",
src/config.ts CHANGED
@@ -23,10 +23,16 @@ export const config = {
23
  headless: process.env.PLAYWRIGHT_HEADLESS !== 'false',
24
 
25
  // Storage buckets
 
26
  inputBucket: process.env.INPUT_BUCKET || 'turnitin-inputs',
27
  reportBucket: process.env.REPORT_BUCKET || 'turnitin-reports',
28
  diagnosticsBucket: process.env.DIAGNOSTICS_BUCKET || 'turnitin-diagnostics',
29
  sessionBucket: process.env.SESSION_BUCKET || 'turnitin-sessions',
 
 
 
 
 
30
 
31
  // Cron intervals (minutes)
32
  quotaCheckInterval: parseInt(process.env.QUOTA_CHECK_INTERVAL || '30', 10),
@@ -70,6 +76,15 @@ export function validateConfig(): void {
70
  ];
71
 
72
  const missing = required.filter((key) => !config[key]);
 
 
 
 
 
 
 
 
 
73
  if (missing.length > 0) {
74
  throw new Error(`Missing required environment variables: ${missing.join(', ')}`);
75
  }
 
23
  headless: process.env.PLAYWRIGHT_HEADLESS !== 'false',
24
 
25
  // Storage buckets
26
+ storageProvider: process.env.STORAGE_PROVIDER === 'r2' ? 'r2' : 'supabase',
27
  inputBucket: process.env.INPUT_BUCKET || 'turnitin-inputs',
28
  reportBucket: process.env.REPORT_BUCKET || 'turnitin-reports',
29
  diagnosticsBucket: process.env.DIAGNOSTICS_BUCKET || 'turnitin-diagnostics',
30
  sessionBucket: process.env.SESSION_BUCKET || 'turnitin-sessions',
31
+ r2AccountId: process.env.R2_ACCOUNT_ID || '',
32
+ r2AccessKeyId: process.env.R2_ACCESS_KEY_ID || '',
33
+ r2SecretAccessKey: process.env.R2_SECRET_ACCESS_KEY || '',
34
+ r2Endpoint: process.env.R2_ENDPOINT || '',
35
+ r2Region: process.env.R2_REGION || 'auto',
36
 
37
  // Cron intervals (minutes)
38
  quotaCheckInterval: parseInt(process.env.QUOTA_CHECK_INTERVAL || '30', 10),
 
76
  ];
77
 
78
  const missing = required.filter((key) => !config[key]);
79
+ if (config.storageProvider === 'r2') {
80
+ const r2Required: (keyof typeof config)[] = [
81
+ 'r2AccountId',
82
+ 'r2AccessKeyId',
83
+ 'r2SecretAccessKey',
84
+ 'r2Endpoint',
85
+ ];
86
+ missing.push(...r2Required.filter((key) => !config[key]));
87
+ }
88
  if (missing.length > 0) {
89
  throw new Error(`Missing required environment variables: ${missing.join(', ')}`);
90
  }
src/db/r2.ts ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import {
2
+ DeleteObjectCommand,
3
+ GetObjectCommand,
4
+ PutObjectCommand,
5
+ S3Client,
6
+ } from '@aws-sdk/client-s3';
7
+ import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
8
+ import * as fs from 'fs';
9
+ import * as path from 'path';
10
+ import { pipeline } from 'stream/promises';
11
+ import { Readable } from 'stream';
12
+ import { config } from '../config';
13
+
14
+ const R2_PREFIX = 'r2://';
15
+ let client: S3Client | null = null;
16
+
17
+ export interface R2ObjectRef {
18
+ bucket: string;
19
+ key: string;
20
+ }
21
+
22
+ function getR2Client(): S3Client {
23
+ if (!client) {
24
+ client = new S3Client({
25
+ region: config.r2Region,
26
+ endpoint: config.r2Endpoint,
27
+ forcePathStyle: true,
28
+ credentials: {
29
+ accessKeyId: config.r2AccessKeyId,
30
+ secretAccessKey: config.r2SecretAccessKey,
31
+ },
32
+ });
33
+ }
34
+ return client;
35
+ }
36
+
37
+ export function isR2ObjectRef(value: string): boolean {
38
+ return value.startsWith(R2_PREFIX);
39
+ }
40
+
41
+ export function createR2ObjectRef(bucket: string, key: string): string {
42
+ return `${R2_PREFIX}${bucket}/${key.replace(/^\/+/, '')}`;
43
+ }
44
+
45
+ export function parseR2ObjectRef(value: string): R2ObjectRef {
46
+ if (!isR2ObjectRef(value)) {
47
+ throw new Error('Not an R2 object reference');
48
+ }
49
+
50
+ const withoutScheme = value.slice(R2_PREFIX.length);
51
+ const slashIndex = withoutScheme.indexOf('/');
52
+ if (slashIndex <= 0 || slashIndex === withoutScheme.length - 1) {
53
+ throw new Error('Invalid R2 object reference');
54
+ }
55
+
56
+ return {
57
+ bucket: withoutScheme.slice(0, slashIndex),
58
+ key: withoutScheme.slice(slashIndex + 1),
59
+ };
60
+ }
61
+
62
+ export async function putR2Buffer(
63
+ bucket: string,
64
+ key: string,
65
+ body: Buffer,
66
+ contentType: string,
67
+ ): Promise<string> {
68
+ await getR2Client().send(
69
+ new PutObjectCommand({
70
+ Bucket: bucket,
71
+ Key: key,
72
+ Body: body,
73
+ ContentLength: body.length,
74
+ ContentType: contentType,
75
+ }),
76
+ );
77
+ return createR2ObjectRef(bucket, key);
78
+ }
79
+
80
+ export async function putR2File(
81
+ bucket: string,
82
+ key: string,
83
+ localPath: string,
84
+ contentType: string,
85
+ ): Promise<string> {
86
+ const stat = await fs.promises.stat(localPath);
87
+ await getR2Client().send(
88
+ new PutObjectCommand({
89
+ Bucket: bucket,
90
+ Key: key,
91
+ Body: fs.createReadStream(localPath),
92
+ ContentLength: stat.size,
93
+ ContentType: contentType,
94
+ }),
95
+ );
96
+ return createR2ObjectRef(bucket, key);
97
+ }
98
+
99
+ export async function downloadR2File(
100
+ objectRef: string,
101
+ localPath: string,
102
+ ): Promise<void> {
103
+ const { bucket, key } = parseR2ObjectRef(objectRef);
104
+ const response = await getR2Client().send(
105
+ new GetObjectCommand({ Bucket: bucket, Key: key }),
106
+ );
107
+ if (!response.Body) throw new Error(`R2 object body is empty: ${objectRef}`);
108
+
109
+ await fs.promises.mkdir(path.dirname(localPath), { recursive: true });
110
+ await pipeline(response.Body as Readable, fs.createWriteStream(localPath));
111
+ }
112
+
113
+ export async function readR2Text(objectRef: string): Promise<string | null> {
114
+ const { bucket, key } = parseR2ObjectRef(objectRef);
115
+ try {
116
+ const response = await getR2Client().send(
117
+ new GetObjectCommand({ Bucket: bucket, Key: key }),
118
+ );
119
+ if (!response.Body) return null;
120
+ return await response.Body.transformToString('utf-8');
121
+ } catch (error: unknown) {
122
+ const name = error instanceof Error ? error.name : '';
123
+ const status = (error as { $metadata?: { httpStatusCode?: number } })?.$metadata
124
+ ?.httpStatusCode;
125
+ if (name === 'NoSuchKey' || name === 'NotFound' || status === 404) return null;
126
+ throw error;
127
+ }
128
+ }
129
+
130
+ export async function deleteR2Object(objectRef: string): Promise<void> {
131
+ const { bucket, key } = parseR2ObjectRef(objectRef);
132
+ await getR2Client().send(
133
+ new DeleteObjectCommand({ Bucket: bucket, Key: key }),
134
+ );
135
+ }
136
+
137
+ export async function createR2SignedDownloadUrl(
138
+ objectRef: string,
139
+ expiresInSeconds: number,
140
+ downloadFileName?: string,
141
+ ): Promise<string> {
142
+ const { bucket, key } = parseR2ObjectRef(objectRef);
143
+ const disposition = downloadFileName
144
+ ? `attachment; filename*=UTF-8''${encodeURIComponent(downloadFileName)}`
145
+ : undefined;
146
+ return getSignedUrl(
147
+ getR2Client(),
148
+ new GetObjectCommand({
149
+ Bucket: bucket,
150
+ Key: key,
151
+ ResponseContentDisposition: disposition,
152
+ }),
153
+ { expiresIn: expiresInSeconds },
154
+ );
155
+ }
src/db/storage.ts CHANGED
@@ -3,6 +3,15 @@ import * as path from 'path';
3
  import { supabase } from './client';
4
  import { config } from '../config';
5
  import { logger } from '../utils/logger';
 
 
 
 
 
 
 
 
 
6
 
7
  /**
8
  * Upload a user's input file to the turnitin-inputs bucket.
@@ -18,6 +27,72 @@ export async function uploadInputFile(
18
  const ext = path.extname(fileName);
19
  const storagePath = `${userId}/${storageKey}/input${ext}`;
20
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
  const { error } = await supabase.storage
22
  .from(config.inputBucket)
23
  .upload(storagePath, fileBuffer, {
@@ -37,6 +112,19 @@ export async function uploadInputFile(
37
  * Download a file from Supabase Storage to a local path.
38
  */
39
  export async function downloadInputFile(storagePath: string, localPath: string): Promise<void> {
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  const { data, error } = await supabase.storage
41
  .from(config.inputBucket)
42
  .download(storagePath);
@@ -54,6 +142,25 @@ export async function downloadInputFile(storagePath: string, localPath: string):
54
  fs.writeFileSync(localPath, buffer);
55
  }
56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  /**
58
  * Upload a generated report PDF to the turnitin-reports bucket.
59
  * Returns the storage path and expiry timestamp.
@@ -64,6 +171,28 @@ export async function uploadReportPdf(
64
  localPdfPath: string,
65
  ): Promise<{ storagePath: string; expiresAt: string }> {
66
  const storagePath = `${userId}/${jobId}/report.pdf`;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  const fileBuffer = fs.readFileSync(localPdfPath);
68
 
69
  const { error } = await supabase.storage
@@ -78,11 +207,7 @@ export async function uploadReportPdf(
78
  throw error;
79
  }
80
 
81
- const expiresAt = new Date(
82
- Date.now() + config.reportRetentionHours * 60 * 60 * 1000,
83
- ).toISOString();
84
-
85
- return { storagePath, expiresAt };
86
  }
87
 
88
  /**
@@ -95,6 +220,28 @@ export async function uploadReceiptPdf(
95
  localPdfPath: string,
96
  ): Promise<{ storagePath: string; expiresAt: string }> {
97
  const storagePath = `${userId}/${jobId}/receipt.pdf`;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  const fileBuffer = fs.readFileSync(localPdfPath);
99
 
100
  const { error } = await supabase.storage
@@ -112,17 +259,26 @@ export async function uploadReceiptPdf(
112
  throw error;
113
  }
114
 
115
- const expiresAt = new Date(
116
- Date.now() + config.reportRetentionHours * 60 * 60 * 1000,
117
- ).toISOString();
118
-
119
- return { storagePath, expiresAt };
120
  }
121
 
122
  /**
123
  * Delete a report PDF from Supabase Storage.
124
  */
125
  export async function deleteReportPdf(storagePath: string): Promise<void> {
 
 
 
 
 
 
 
 
 
 
 
 
 
126
  const { error } = await supabase.storage
127
  .from(config.reportBucket)
128
  .remove([storagePath]);
@@ -143,6 +299,23 @@ export async function uploadStorageState(
143
  ): Promise<string> {
144
  const storagePath = `${accountId}/state.json`;
145
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
146
  const { error } = await supabase.storage
147
  .from(config.sessionBucket)
148
  .upload(storagePath, Buffer.from(stateJson, 'utf-8'), {
@@ -163,6 +336,18 @@ export async function uploadStorageState(
163
  * Returns the JSON string, or null if not found.
164
  */
165
  export async function downloadStorageState(storagePath: string): Promise<string | null> {
 
 
 
 
 
 
 
 
 
 
 
 
166
  const { data, error } = await supabase.storage
167
  .from(config.sessionBucket)
168
  .download(storagePath);
@@ -188,6 +373,23 @@ export async function createSignedUrl(
188
  expiresInSeconds: number,
189
  downloadFileName?: string,
190
  ): Promise<string> {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
191
  const { data, error } = await supabase.storage
192
  .from(bucket)
193
  .createSignedUrl(
@@ -204,6 +406,12 @@ export async function createSignedUrl(
204
  return data.signedUrl;
205
  }
206
 
 
 
 
 
 
 
207
  /** Map file extensions to MIME types */
208
  function getMimeType(ext: string): string {
209
  const mimeTypes: Record<string, string> = {
 
3
  import { supabase } from './client';
4
  import { config } from '../config';
5
  import { logger } from '../utils/logger';
6
+ import {
7
+ createR2SignedDownloadUrl,
8
+ deleteR2Object,
9
+ downloadR2File,
10
+ isR2ObjectRef,
11
+ putR2Buffer,
12
+ putR2File,
13
+ readR2Text,
14
+ } from './r2';
15
 
16
  /**
17
  * Upload a user's input file to the turnitin-inputs bucket.
 
27
  const ext = path.extname(fileName);
28
  const storagePath = `${userId}/${storageKey}/input${ext}`;
29
 
30
+ if (config.storageProvider === 'r2') {
31
+ try {
32
+ return await putR2Buffer(
33
+ config.inputBucket,
34
+ storagePath,
35
+ fileBuffer,
36
+ getMimeType(ext),
37
+ );
38
+ } catch (error) {
39
+ logger.error('Failed to upload input file to R2', {
40
+ storagePath,
41
+ error: error instanceof Error ? error.message : String(error),
42
+ });
43
+ throw error;
44
+ }
45
+ }
46
+
47
+ const { error } = await supabase.storage
48
+ .from(config.inputBucket)
49
+ .upload(storagePath, fileBuffer, {
50
+ contentType: getMimeType(ext),
51
+ upsert,
52
+ });
53
+
54
+ if (error) {
55
+ logger.error('Failed to upload input file', { storagePath, error: error.message });
56
+ throw error;
57
+ }
58
+
59
+ return storagePath;
60
+ }
61
+
62
+ /**
63
+ * Upload an input file from disk. R2 receives a stream so a 100 MB user upload
64
+ * does not need a second full-size in-memory copy inside the worker.
65
+ */
66
+ export async function uploadInputFileFromPath(
67
+ userId: string,
68
+ storageKey: string,
69
+ fileName: string,
70
+ localPath: string,
71
+ upsert = false,
72
+ ): Promise<string> {
73
+ const ext = path.extname(fileName);
74
+ const storagePath = `${userId}/${storageKey}/input${ext}`;
75
+
76
+ if (config.storageProvider === 'r2') {
77
+ try {
78
+ return await putR2File(
79
+ config.inputBucket,
80
+ storagePath,
81
+ localPath,
82
+ getMimeType(ext),
83
+ );
84
+ } catch (error) {
85
+ logger.error('Failed to upload input file to R2', {
86
+ storagePath,
87
+ error: error instanceof Error ? error.message : String(error),
88
+ });
89
+ throw error;
90
+ }
91
+ }
92
+
93
+ // Supabase Storage's Node client expects a buffer. This fallback only serves
94
+ // legacy objects/deployments; R2 is the configured destination for new jobs.
95
+ const fileBuffer = await fs.promises.readFile(localPath);
96
  const { error } = await supabase.storage
97
  .from(config.inputBucket)
98
  .upload(storagePath, fileBuffer, {
 
112
  * Download a file from Supabase Storage to a local path.
113
  */
114
  export async function downloadInputFile(storagePath: string, localPath: string): Promise<void> {
115
+ if (isR2ObjectRef(storagePath)) {
116
+ try {
117
+ await downloadR2File(storagePath, localPath);
118
+ return;
119
+ } catch (error) {
120
+ logger.error('Failed to download input file from R2', {
121
+ storagePath,
122
+ error: error instanceof Error ? error.message : String(error),
123
+ });
124
+ throw error;
125
+ }
126
+ }
127
+
128
  const { data, error } = await supabase.storage
129
  .from(config.inputBucket)
130
  .download(storagePath);
 
142
  fs.writeFileSync(localPath, buffer);
143
  }
144
 
145
+ /** Delete a staged input when atomic job creation definitively did not happen. */
146
+ export async function deleteInputFile(storagePath: string): Promise<void> {
147
+ if (isR2ObjectRef(storagePath)) {
148
+ await deleteR2Object(storagePath);
149
+ return;
150
+ }
151
+
152
+ const { error } = await supabase.storage
153
+ .from(config.inputBucket)
154
+ .remove([storagePath]);
155
+ if (error) {
156
+ logger.error('Failed to delete staged input file', {
157
+ storagePath,
158
+ error: error.message,
159
+ });
160
+ throw error;
161
+ }
162
+ }
163
+
164
  /**
165
  * Upload a generated report PDF to the turnitin-reports bucket.
166
  * Returns the storage path and expiry timestamp.
 
171
  localPdfPath: string,
172
  ): Promise<{ storagePath: string; expiresAt: string }> {
173
  const storagePath = `${userId}/${jobId}/report.pdf`;
174
+
175
+ if (config.storageProvider === 'r2') {
176
+ try {
177
+ const objectRef = await putR2File(
178
+ config.reportBucket,
179
+ storagePath,
180
+ localPdfPath,
181
+ 'application/pdf',
182
+ );
183
+ return {
184
+ storagePath: objectRef,
185
+ expiresAt: createReportExpiry(),
186
+ };
187
+ } catch (error) {
188
+ logger.error('Failed to upload report PDF to R2', {
189
+ storagePath,
190
+ error: error instanceof Error ? error.message : String(error),
191
+ });
192
+ throw error;
193
+ }
194
+ }
195
+
196
  const fileBuffer = fs.readFileSync(localPdfPath);
197
 
198
  const { error } = await supabase.storage
 
207
  throw error;
208
  }
209
 
210
+ return { storagePath, expiresAt: createReportExpiry() };
 
 
 
 
211
  }
212
 
213
  /**
 
220
  localPdfPath: string,
221
  ): Promise<{ storagePath: string; expiresAt: string }> {
222
  const storagePath = `${userId}/${jobId}/receipt.pdf`;
223
+
224
+ if (config.storageProvider === 'r2') {
225
+ try {
226
+ const objectRef = await putR2File(
227
+ config.reportBucket,
228
+ storagePath,
229
+ localPdfPath,
230
+ 'application/pdf',
231
+ );
232
+ return {
233
+ storagePath: objectRef,
234
+ expiresAt: createReportExpiry(),
235
+ };
236
+ } catch (error) {
237
+ logger.error('Failed to upload Digital Receipt PDF to R2', {
238
+ storagePath,
239
+ error: error instanceof Error ? error.message : String(error),
240
+ });
241
+ throw error;
242
+ }
243
+ }
244
+
245
  const fileBuffer = fs.readFileSync(localPdfPath);
246
 
247
  const { error } = await supabase.storage
 
259
  throw error;
260
  }
261
 
262
+ return { storagePath, expiresAt: createReportExpiry() };
 
 
 
 
263
  }
264
 
265
  /**
266
  * Delete a report PDF from Supabase Storage.
267
  */
268
  export async function deleteReportPdf(storagePath: string): Promise<void> {
269
+ if (isR2ObjectRef(storagePath)) {
270
+ try {
271
+ await deleteR2Object(storagePath);
272
+ return;
273
+ } catch (error) {
274
+ logger.error('Failed to delete report PDF from R2', {
275
+ storagePath,
276
+ error: error instanceof Error ? error.message : String(error),
277
+ });
278
+ throw error;
279
+ }
280
+ }
281
+
282
  const { error } = await supabase.storage
283
  .from(config.reportBucket)
284
  .remove([storagePath]);
 
299
  ): Promise<string> {
300
  const storagePath = `${accountId}/state.json`;
301
 
302
+ if (config.storageProvider === 'r2') {
303
+ try {
304
+ return await putR2Buffer(
305
+ config.sessionBucket,
306
+ storagePath,
307
+ Buffer.from(stateJson, 'utf-8'),
308
+ 'application/json',
309
+ );
310
+ } catch (error) {
311
+ logger.error('Failed to upload storage state to R2', {
312
+ accountId,
313
+ error: error instanceof Error ? error.message : String(error),
314
+ });
315
+ throw error;
316
+ }
317
+ }
318
+
319
  const { error } = await supabase.storage
320
  .from(config.sessionBucket)
321
  .upload(storagePath, Buffer.from(stateJson, 'utf-8'), {
 
336
  * Returns the JSON string, or null if not found.
337
  */
338
  export async function downloadStorageState(storagePath: string): Promise<string | null> {
339
+ if (isR2ObjectRef(storagePath)) {
340
+ try {
341
+ return await readR2Text(storagePath);
342
+ } catch (error) {
343
+ logger.error('Failed to download storage state from R2', {
344
+ storagePath,
345
+ error: error instanceof Error ? error.message : String(error),
346
+ });
347
+ throw error;
348
+ }
349
+ }
350
+
351
  const { data, error } = await supabase.storage
352
  .from(config.sessionBucket)
353
  .download(storagePath);
 
373
  expiresInSeconds: number,
374
  downloadFileName?: string,
375
  ): Promise<string> {
376
+ if (isR2ObjectRef(filePath)) {
377
+ try {
378
+ return await createR2SignedDownloadUrl(
379
+ filePath,
380
+ expiresInSeconds,
381
+ downloadFileName,
382
+ );
383
+ } catch (error) {
384
+ logger.error('Failed to create R2 signed URL', {
385
+ bucket,
386
+ filePath,
387
+ error: error instanceof Error ? error.message : String(error),
388
+ });
389
+ throw error;
390
+ }
391
+ }
392
+
393
  const { data, error } = await supabase.storage
394
  .from(bucket)
395
  .createSignedUrl(
 
406
  return data.signedUrl;
407
  }
408
 
409
+ function createReportExpiry(): string {
410
+ return new Date(
411
+ Date.now() + config.reportRetentionHours * 60 * 60 * 1000,
412
+ ).toISOString();
413
+ }
414
+
415
  /** Map file extensions to MIME types */
416
  function getMimeType(ext: string): string {
417
  const mimeTypes: Record<string, string> = {
src/engine/legacy.ts CHANGED
@@ -12,11 +12,22 @@ import type { FilterOptions } from './steps/filters';
12
  import type { RunTurnitinJobInput, RunTurnitinJobResult } from './turnitin';
13
  import type { SubmissionDetails } from './steps/submission-details';
14
  import type { Frame } from 'playwright';
 
15
 
16
  type Scope = Page | Frame;
17
 
18
  type EngineEvent = NonNullable<RunTurnitinJobInput['onEvent']>;
19
  const LEGACY_ACCOUNT_QUOTA_LIMIT = 4;
 
 
 
 
 
 
 
 
 
 
20
 
21
  async function emit(
22
  onEvent: RunTurnitinJobInput['onEvent'],
@@ -50,10 +61,16 @@ function parseNumber(value: string | null | undefined): number | undefined {
50
  }
51
 
52
  function parsePercent(value: string | null | undefined): number | null {
53
- const match = String(value || '').match(/(\d{1,3})\s*%?/);
54
- if (!match) return null;
55
- const parsed = Number(match[1]);
56
- return Number.isFinite(parsed) ? parsed : null;
 
 
 
 
 
 
57
  }
58
 
59
  function parseFileSizeBytes(value: string | null | undefined): number | undefined {
@@ -68,7 +85,12 @@ function parseFileSizeBytes(value: string | null | undefined): number | undefine
68
  return Math.round(amount);
69
  }
70
 
71
- function normalizeLabel(label: string): keyof SubmissionDetails | null {
 
 
 
 
 
72
  const normalized = label.toLowerCase().replace(/\s+/g, ' ').trim();
73
  if (normalized === 'student id') return 'studentId';
74
  if (normalized === 'class name') return 'className';
@@ -581,9 +603,11 @@ async function completeLegacyUpload(
581
  return reviewDetails;
582
  }
583
 
584
- async function waitForLegacySimilarity(page: Page): Promise<number | null> {
 
 
 
585
  const startedAt = Date.now();
586
- const timeoutMs = Math.max(config.similarityTimeoutMs || 180000, 120000);
587
  const reloadIntervalMs = 30000; // Reload every 30 seconds
588
  let lastReloadAt = startedAt;
589
 
@@ -635,7 +659,11 @@ function isViewerUrl(url: string): boolean {
635
  return VIEWER_URL_PATTERNS.some((re) => re.test(url));
636
  }
637
 
638
- async function openLegacyViewer(page: Page, context: BrowserContext): Promise<Page> {
 
 
 
 
639
  const opener = page
640
  .locator('td[data-title="Similarity Score"] a.similarity-open, a.default-open[data-paper-title], a.default-open')
641
  .first();
@@ -663,11 +691,12 @@ async function openLegacyViewer(page: Page, context: BrowserContext): Promise<Pa
663
  throw new Error(`Legacy report viewer did not open. Current URL: ${viewerPage.url()}`);
664
  }
665
 
666
- // Wait for the carta viewer sidebar to render filter controls (up to 20 s).
 
667
  await visible(
668
  viewerPage,
669
  '.apply-changes-button, .osi-score, .sidebar-paper-info-button, .exclude-quotes-checkbox',
670
- 20000,
671
  ).catch(() => {});
672
 
673
  return viewerPage;
@@ -1700,6 +1729,52 @@ export async function runLegacyTurnitinJob(
1700
 
1701
  const result: RunTurnitinJobResult = {};
1702
  let page: Page | null = null;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1703
  const browser = await getBrowser();
1704
  const contextOptions: Record<string, unknown> = {
1705
  userAgent:
@@ -1775,8 +1850,17 @@ export async function runLegacyTurnitinJob(
1775
  });
1776
  const reviewDetails = await completeLegacyUpload(scope, inputFilePath, inputFileName || path.basename(inputFilePath));
1777
  if (reviewDetails) {
1778
- result.submissionDetails = reviewDetails;
1779
- await emit(onEvent, 'info', 'submission_details', 'Legacy review details captured', reviewDetails as Record<string, unknown>);
 
 
 
 
 
 
 
 
 
1780
  }
1781
  result.submittedAt = new Date().toISOString();
1782
  result.lastCompletedStep = 'submitted';
@@ -1786,15 +1870,27 @@ export async function runLegacyTurnitinJob(
1786
  }
1787
 
1788
  await emit(onEvent, 'info', 'similarity', 'Waiting for legacy similarity score');
1789
- const tableSimilarity = await waitForLegacySimilarity(page);
1790
  if (tableSimilarity !== null) result.similarityPercent = tableSimilarity;
1791
  result.lastCompletedStep = 'similarity';
1792
  await emit(onEvent, 'info', 'similarity', `Similarity: ${result.similarityPercent ?? 'pending'}%`, {
1793
  similarityPercent: result.similarityPercent,
 
 
1794
  });
1795
 
 
 
 
 
 
 
1796
  await emit(onEvent, 'info', 'viewer', 'Opening legacy report viewer');
1797
- page = await openLegacyViewer(page, context);
 
 
 
 
1798
  result.viewerUrl = page.url();
1799
  result.lastCompletedStep = 'viewer';
1800
  await emit(onEvent, 'info', 'viewer', 'Legacy report viewer opened', {
 
12
  import type { RunTurnitinJobInput, RunTurnitinJobResult } from './turnitin';
13
  import type { SubmissionDetails } from './steps/submission-details';
14
  import type { Frame } from 'playwright';
15
+ import { readLocalDocumentMetadata } from '../utils/document-metadata';
16
 
17
  type Scope = Page | Frame;
18
 
19
  type EngineEvent = NonNullable<RunTurnitinJobInput['onEvent']>;
20
  const LEGACY_ACCOUNT_QUOTA_LIMIT = 4;
21
+ const LARGE_DOCUMENT_PAGE_THRESHOLD = 100;
22
+
23
+ function getLegacySimilarityTimeoutMs(pageCount?: number): number {
24
+ const baseTimeout = Math.max(config.similarityTimeoutMs || 180000, 120000);
25
+ if (!pageCount || pageCount < LARGE_DOCUMENT_PAGE_THRESHOLD) return baseTimeout;
26
+
27
+ const extraHundreds = Math.max(0, Math.floor((pageCount - 100) / 100));
28
+ const largeDocumentMinutes = Math.min(15, 8 + extraHundreds * 2);
29
+ return Math.max(baseTimeout, largeDocumentMinutes * 60 * 1000);
30
+ }
31
 
32
  async function emit(
33
  onEvent: RunTurnitinJobInput['onEvent'],
 
61
  }
62
 
63
  function parsePercent(value: string | null | undefined): number | null {
64
+ const text = String(value || '').trim();
65
+ const explicitPercent = text.match(/(?:^|\D)(\d{1,3})\s*%/);
66
+ const numericOnly = text.match(/^\d{1,3}$/);
67
+ const raw = explicitPercent?.[1] || numericOnly?.[0];
68
+ if (!raw) return null;
69
+
70
+ const parsed = Number(raw);
71
+ return Number.isInteger(parsed) && parsed >= 0 && parsed <= 100
72
+ ? parsed
73
+ : null;
74
  }
75
 
76
  function parseFileSizeBytes(value: string | null | undefined): number | undefined {
 
85
  return Math.round(amount);
86
  }
87
 
88
+ type ParsedLegacySubmissionDetailKey = Exclude<
89
+ keyof SubmissionDetails,
90
+ 'pageCountSource' | 'expectedWaitMinutes' | 'largeDocument'
91
+ >;
92
+
93
+ function normalizeLabel(label: string): ParsedLegacySubmissionDetailKey | null {
94
  const normalized = label.toLowerCase().replace(/\s+/g, ' ').trim();
95
  if (normalized === 'student id') return 'studentId';
96
  if (normalized === 'class name') return 'className';
 
603
  return reviewDetails;
604
  }
605
 
606
+ async function waitForLegacySimilarity(
607
+ page: Page,
608
+ timeoutMs: number,
609
+ ): Promise<number | null> {
610
  const startedAt = Date.now();
 
611
  const reloadIntervalMs = 30000; // Reload every 30 seconds
612
  let lastReloadAt = startedAt;
613
 
 
659
  return VIEWER_URL_PATTERNS.some((re) => re.test(url));
660
  }
661
 
662
+ async function openLegacyViewer(
663
+ page: Page,
664
+ context: BrowserContext,
665
+ readyTimeoutMs = 20000,
666
+ ): Promise<Page> {
667
  const opener = page
668
  .locator('td[data-title="Similarity Score"] a.similarity-open, a.default-open[data-paper-title], a.default-open')
669
  .first();
 
691
  throw new Error(`Legacy report viewer did not open. Current URL: ${viewerPage.url()}`);
692
  }
693
 
694
+ // Large documents may finish the dashboard score before the Carta sidebar
695
+ // has fully hydrated, so the caller can grant a longer readiness window.
696
  await visible(
697
  viewerPage,
698
  '.apply-changes-button, .osi-score, .sidebar-paper-info-button, .exclude-quotes-checkbox',
699
+ readyTimeoutMs,
700
  ).catch(() => {});
701
 
702
  return viewerPage;
 
1729
 
1730
  const result: RunTurnitinJobResult = {};
1731
  let page: Page | null = null;
1732
+ const localDocumentMetadata =
1733
+ requestedMode === 'quota_check'
1734
+ ? {}
1735
+ : await readLocalDocumentMetadata(inputFilePath);
1736
+ const localPageCount = localDocumentMetadata.pageCount;
1737
+ const largeDocument = Boolean(
1738
+ localPageCount && localPageCount >= LARGE_DOCUMENT_PAGE_THRESHOLD,
1739
+ );
1740
+ const similarityTimeoutMs = getLegacySimilarityTimeoutMs(localPageCount);
1741
+ const expectedWaitMinutes = Math.ceil(similarityTimeoutMs / 60000);
1742
+
1743
+ if (localPageCount) {
1744
+ result.submissionDetails = {
1745
+ pageCount: localPageCount,
1746
+ pageCountSource: localDocumentMetadata.pageCountSource,
1747
+ expectedWaitMinutes,
1748
+ largeDocument,
1749
+ };
1750
+ await emit(
1751
+ onEvent,
1752
+ 'info',
1753
+ 'submission_details',
1754
+ `Document page count detected: ${localPageCount} pages.`,
1755
+ {
1756
+ pageCount: localPageCount,
1757
+ pageCountSource: localDocumentMetadata.pageCountSource,
1758
+ expectedWaitMinutes,
1759
+ largeDocument,
1760
+ },
1761
+ );
1762
+ if (largeDocument) {
1763
+ await emit(
1764
+ onEvent,
1765
+ 'warning',
1766
+ 'large_document',
1767
+ `Large document detected: ${localPageCount} pages. Similarity processing may take up to ${expectedWaitMinutes} minutes.`,
1768
+ {
1769
+ pageCount: localPageCount,
1770
+ pageCountSource: localDocumentMetadata.pageCountSource,
1771
+ expectedWaitMinutes,
1772
+ largeDocument: true,
1773
+ },
1774
+ );
1775
+ }
1776
+ }
1777
+
1778
  const browser = await getBrowser();
1779
  const contextOptions: Record<string, unknown> = {
1780
  userAgent:
 
1850
  });
1851
  const reviewDetails = await completeLegacyUpload(scope, inputFilePath, inputFileName || path.basename(inputFilePath));
1852
  if (reviewDetails) {
1853
+ result.submissionDetails = {
1854
+ ...(result.submissionDetails || {}),
1855
+ ...reviewDetails,
1856
+ };
1857
+ await emit(
1858
+ onEvent,
1859
+ 'info',
1860
+ 'submission_details',
1861
+ 'Legacy review details captured',
1862
+ result.submissionDetails as Record<string, unknown>,
1863
+ );
1864
  }
1865
  result.submittedAt = new Date().toISOString();
1866
  result.lastCompletedStep = 'submitted';
 
1870
  }
1871
 
1872
  await emit(onEvent, 'info', 'similarity', 'Waiting for legacy similarity score');
1873
+ const tableSimilarity = await waitForLegacySimilarity(page, similarityTimeoutMs);
1874
  if (tableSimilarity !== null) result.similarityPercent = tableSimilarity;
1875
  result.lastCompletedStep = 'similarity';
1876
  await emit(onEvent, 'info', 'similarity', `Similarity: ${result.similarityPercent ?? 'pending'}%`, {
1877
  similarityPercent: result.similarityPercent,
1878
+ pageCount: localPageCount,
1879
+ timeoutMs: similarityTimeoutMs,
1880
  });
1881
 
1882
+ if (tableSimilarity === null) {
1883
+ throw new Error(
1884
+ `Legacy similarity is still processing after ${expectedWaitMinutes} minutes. Retrying from the similarity checkpoint without uploading the file again.`,
1885
+ );
1886
+ }
1887
+
1888
  await emit(onEvent, 'info', 'viewer', 'Opening legacy report viewer');
1889
+ page = await openLegacyViewer(
1890
+ page,
1891
+ context,
1892
+ largeDocument ? 60000 : 20000,
1893
+ );
1894
  result.viewerUrl = page.url();
1895
  result.lastCompletedStep = 'viewer';
1896
  await emit(onEvent, 'info', 'viewer', 'Legacy report viewer opened', {
src/engine/steps/submission-details.ts CHANGED
@@ -17,9 +17,17 @@ export interface SubmissionDetails {
17
  charCount?: number;
18
  wordCount?: number;
19
  pageCount?: number;
 
 
 
20
  }
21
 
22
- function normalizeKey(label: string): keyof SubmissionDetails | null {
 
 
 
 
 
23
  const normalized = label.toLowerCase().replace(/\s+/g, ' ').trim();
24
  if (normalized === 'student id') return 'studentId';
25
  if (normalized === 'class name') return 'className';
 
17
  charCount?: number;
18
  wordCount?: number;
19
  pageCount?: number;
20
+ pageCountSource?: 'pdf' | 'docx_metadata';
21
+ expectedWaitMinutes?: number;
22
+ largeDocument?: boolean;
23
  }
24
 
25
+ type ParsedSubmissionDetailKey = Exclude<
26
+ keyof SubmissionDetails,
27
+ 'pageCountSource' | 'expectedWaitMinutes' | 'largeDocument'
28
+ >;
29
+
30
+ function normalizeKey(label: string): ParsedSubmissionDetailKey | null {
31
  const normalized = label.toLowerCase().replace(/\s+/g, ' ').trim();
32
  if (normalized === 'student id') return 'studentId';
33
  if (normalized === 'class name') return 'className';
src/server/routes/submit.ts CHANGED
@@ -1,7 +1,7 @@
1
- import { Router, Response } from 'express';
2
  import multer from 'multer';
3
  import { authenticateUser, AuthenticatedRequest } from '../middleware/auth';
4
- import { uploadInputFile } from '../../db/storage';
5
  import {
6
  createJobWithTicket,
7
  getJobBySubmissionRequestId,
@@ -9,15 +9,43 @@ import {
9
  } from '../../db/tickets';
10
  import { logger } from '../../utils/logger';
11
  import { createHash, randomUUID } from 'crypto';
 
 
12
 
13
  const router = Router();
14
 
15
- /** Multer configured for in-memory storage (files go to Supabase, not local disk) */
 
 
 
16
  const upload = multer({
17
- storage: multer.memoryStorage(),
18
- limits: { fileSize: 50 * 1024 * 1024 }, // 50 MB
 
 
 
 
 
19
  });
20
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
  /** Allowed file extensions for Turnitin submissions */
22
  const ALLOWED_EXTENSIONS = new Set([
23
  '.docx',
@@ -48,6 +76,14 @@ function getFileExtension(filename: string): string {
48
  return lastDot >= 0 ? filename.slice(lastDot).toLowerCase() : '';
49
  }
50
 
 
 
 
 
 
 
 
 
51
  function existingJobMatchesRequest(
52
  existingJob: {
53
  assignment_target_id: string;
@@ -81,9 +117,10 @@ function existingJobMatchesRequest(
81
  router.post(
82
  '/api/submit',
83
  authenticateUser,
84
- upload.single('file'),
85
  async (req, res: Response): Promise<void> => {
86
  const authReq = req as AuthenticatedRequest;
 
87
 
88
  try {
89
  // 1. Validate required fields
@@ -138,9 +175,10 @@ router.post(
138
  res.status(400).json({ error: 'File is required' });
139
  return;
140
  }
 
141
 
142
  // 3. Validate file type
143
- const ext = getFileExtension(authReq.file.originalname);
144
  if (!ALLOWED_EXTENSIONS.has(ext)) {
145
  res.status(400).json({
146
  error: `Unsupported file type: ${ext}. Allowed: ${[...ALLOWED_EXTENSIONS].join(', ')}`,
@@ -170,9 +208,7 @@ router.post(
170
  }
171
 
172
  // 5. Fingerprint the payload before any state-changing operation.
173
- const inputFileSha256 = createHash('sha256')
174
- .update(authReq.file.buffer)
175
- .digest('hex');
176
 
177
  const existingJob = await getJobBySubmissionRequestId(
178
  authReq.userId,
@@ -183,8 +219,8 @@ router.post(
183
  if (!existingJobMatchesRequest(existingJob, {
184
  assignmentTargetId: String(assignment_target_id),
185
  mode: String(mode),
186
- inputFileName: authReq.file.originalname,
187
- inputFileSize: authReq.file.size,
188
  inputFileSha256,
189
  })) {
190
  res.status(409).json({
@@ -209,27 +245,81 @@ router.post(
209
 
210
  // A deterministic request/hash path makes concurrent retries upload the
211
  // same bytes to the same object before the database invariant resolves them.
212
- const storagePath = await uploadInputFile(
213
  authReq.userId,
214
  `${submissionRequestId}/${inputFileSha256}`,
215
- authReq.file.originalname,
216
- authReq.file.buffer,
217
  true,
218
  );
219
 
220
  // 6. Atomically create one job/ticket ledger entry, or return the job
221
  // already created by another Space handling this exact request.
222
- const creation = await createJobWithTicket({
223
- userId: authReq.userId,
224
- assignmentTargetId: assignment_target_id,
225
- mode,
226
- filters,
227
- inputFileName: authReq.file.originalname,
228
- inputStoragePath: storagePath,
229
- inputFileSize: authReq.file.size,
230
- inputFileSha256,
231
- submissionRequestId,
232
- });
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
233
 
234
  // Fetch updated ticket balance
235
  const profile = await getUserProfile(authReq.userId);
@@ -238,7 +328,7 @@ router.post(
238
  logger.info(creation.created ? 'Job submitted successfully' : 'Idempotent submit race resolved', {
239
  jobId: creation.jobId,
240
  userId: authReq.userId,
241
- fileName: authReq.file.originalname,
242
  submissionRequestId,
243
  });
244
 
@@ -268,6 +358,15 @@ router.post(
268
  });
269
 
270
  res.status(500).json({ error: 'Internal server error' });
 
 
 
 
 
 
 
 
 
271
  }
272
  },
273
  );
 
1
+ import { NextFunction, Request, Router, Response } from 'express';
2
  import multer from 'multer';
3
  import { authenticateUser, AuthenticatedRequest } from '../middleware/auth';
4
+ import { deleteInputFile, uploadInputFileFromPath } from '../../db/storage';
5
  import {
6
  createJobWithTicket,
7
  getJobBySubmissionRequestId,
 
9
  } from '../../db/tickets';
10
  import { logger } from '../../utils/logger';
11
  import { createHash, randomUUID } from 'crypto';
12
+ import { createReadStream, promises as fsPromises } from 'fs';
13
+ import * as os from 'os';
14
 
15
  const router = Router();
16
 
17
+ /**
18
+ * Keep large uploads out of the worker heap. The route removes every temporary
19
+ * file after validation/storage, including early returns and failed requests.
20
+ */
21
  const upload = multer({
22
+ storage: multer.diskStorage({
23
+ destination: os.tmpdir(),
24
+ filename: (_req, file, callback) => {
25
+ callback(null, `relv-upload-${randomUUID()}${getFileExtension(file.originalname)}`);
26
+ },
27
+ }),
28
+ limits: { fileSize: 100 * 1024 * 1024 }, // User uploads are capped at 100 MB
29
  });
30
 
31
+ function receiveUpload(
32
+ req: Request,
33
+ res: Response,
34
+ next: NextFunction,
35
+ ): void {
36
+ upload.single('file')(req, res, (error: unknown) => {
37
+ if (error instanceof multer.MulterError && error.code === 'LIMIT_FILE_SIZE') {
38
+ res.status(413).json({ error: 'File exceeds the 100MB upload limit' });
39
+ return;
40
+ }
41
+ if (error) {
42
+ next(error);
43
+ return;
44
+ }
45
+ next();
46
+ });
47
+ }
48
+
49
  /** Allowed file extensions for Turnitin submissions */
50
  const ALLOWED_EXTENSIONS = new Set([
51
  '.docx',
 
76
  return lastDot >= 0 ? filename.slice(lastDot).toLowerCase() : '';
77
  }
78
 
79
+ async function sha256File(filePath: string): Promise<string> {
80
+ const hash = createHash('sha256');
81
+ for await (const chunk of createReadStream(filePath)) {
82
+ hash.update(chunk as Buffer);
83
+ }
84
+ return hash.digest('hex');
85
+ }
86
+
87
  function existingJobMatchesRequest(
88
  existingJob: {
89
  assignment_target_id: string;
 
117
  router.post(
118
  '/api/submit',
119
  authenticateUser,
120
+ receiveUpload,
121
  async (req, res: Response): Promise<void> => {
122
  const authReq = req as AuthenticatedRequest;
123
+ const temporaryUploadPath = authReq.file?.path;
124
 
125
  try {
126
  // 1. Validate required fields
 
175
  res.status(400).json({ error: 'File is required' });
176
  return;
177
  }
178
+ const uploadedFile = authReq.file;
179
 
180
  // 3. Validate file type
181
+ const ext = getFileExtension(uploadedFile.originalname);
182
  if (!ALLOWED_EXTENSIONS.has(ext)) {
183
  res.status(400).json({
184
  error: `Unsupported file type: ${ext}. Allowed: ${[...ALLOWED_EXTENSIONS].join(', ')}`,
 
208
  }
209
 
210
  // 5. Fingerprint the payload before any state-changing operation.
211
+ const inputFileSha256 = await sha256File(uploadedFile.path);
 
 
212
 
213
  const existingJob = await getJobBySubmissionRequestId(
214
  authReq.userId,
 
219
  if (!existingJobMatchesRequest(existingJob, {
220
  assignmentTargetId: String(assignment_target_id),
221
  mode: String(mode),
222
+ inputFileName: uploadedFile.originalname,
223
+ inputFileSize: uploadedFile.size,
224
  inputFileSha256,
225
  })) {
226
  res.status(409).json({
 
245
 
246
  // A deterministic request/hash path makes concurrent retries upload the
247
  // same bytes to the same object before the database invariant resolves them.
248
+ const storagePath = await uploadInputFileFromPath(
249
  authReq.userId,
250
  `${submissionRequestId}/${inputFileSha256}`,
251
+ uploadedFile.originalname,
252
+ uploadedFile.path,
253
  true,
254
  );
255
 
256
  // 6. Atomically create one job/ticket ledger entry, or return the job
257
  // already created by another Space handling this exact request.
258
+ const creation = await (async () => {
259
+ try {
260
+ return await createJobWithTicket({
261
+ userId: authReq.userId,
262
+ assignmentTargetId: assignment_target_id,
263
+ mode,
264
+ filters,
265
+ inputFileName: uploadedFile.originalname,
266
+ inputStoragePath: storagePath,
267
+ inputFileSize: uploadedFile.size,
268
+ inputFileSha256,
269
+ submissionRequestId,
270
+ });
271
+ } catch (creationError) {
272
+ // The RPC response may fail after the transaction commits. Confirm
273
+ // database state before deleting the staged object.
274
+ let recoveredJob;
275
+ try {
276
+ recoveredJob = await getJobBySubmissionRequestId(
277
+ authReq.userId,
278
+ submissionRequestId,
279
+ );
280
+ } catch (recoveryError) {
281
+ logger.warn('Could not verify job creation after RPC failure; preserving input object', {
282
+ userId: authReq.userId,
283
+ submissionRequestId,
284
+ error:
285
+ recoveryError instanceof Error
286
+ ? recoveryError.message
287
+ : String(recoveryError),
288
+ });
289
+ throw creationError;
290
+ }
291
+
292
+ if (recoveredJob) {
293
+ if (!existingJobMatchesRequest(recoveredJob, {
294
+ assignmentTargetId: String(assignment_target_id),
295
+ mode: String(mode),
296
+ inputFileName: uploadedFile.originalname,
297
+ inputFileSize: uploadedFile.size,
298
+ inputFileSha256,
299
+ })) {
300
+ await deleteInputFile(storagePath).catch(() => {});
301
+ throw new Error('Idempotency key recovered a different job payload');
302
+ }
303
+ logger.warn('Recovered committed job after job-creation response failure', {
304
+ jobId: recoveredJob.id,
305
+ userId: authReq.userId,
306
+ submissionRequestId,
307
+ });
308
+ return { jobId: recoveredJob.id, created: false };
309
+ }
310
+
311
+ await deleteInputFile(storagePath).catch((cleanupError: unknown) => {
312
+ logger.warn('Failed to roll back staged input after job creation failure', {
313
+ storagePath,
314
+ error:
315
+ cleanupError instanceof Error
316
+ ? cleanupError.message
317
+ : String(cleanupError),
318
+ });
319
+ });
320
+ throw creationError;
321
+ }
322
+ })();
323
 
324
  // Fetch updated ticket balance
325
  const profile = await getUserProfile(authReq.userId);
 
328
  logger.info(creation.created ? 'Job submitted successfully' : 'Idempotent submit race resolved', {
329
  jobId: creation.jobId,
330
  userId: authReq.userId,
331
+ fileName: uploadedFile.originalname,
332
  submissionRequestId,
333
  });
334
 
 
358
  });
359
 
360
  res.status(500).json({ error: 'Internal server error' });
361
+ } finally {
362
+ if (temporaryUploadPath) {
363
+ await fsPromises.unlink(temporaryUploadPath).catch((error: unknown) => {
364
+ logger.warn('Failed to remove temporary upload file', {
365
+ temporaryUploadPath,
366
+ error: error instanceof Error ? error.message : String(error),
367
+ });
368
+ });
369
+ }
370
  }
371
  },
372
  );
src/utils/document-metadata.ts ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import * as fs from 'fs';
2
+ import * as path from 'path';
3
+ import AdmZip from 'adm-zip';
4
+ import { PDFDocument } from 'pdf-lib';
5
+ import { logger } from './logger';
6
+
7
+ export interface LocalDocumentMetadata {
8
+ pageCount?: number;
9
+ pageCountSource?: 'pdf' | 'docx_metadata';
10
+ }
11
+
12
+ export async function readLocalDocumentMetadata(
13
+ filePath: string,
14
+ ): Promise<LocalDocumentMetadata> {
15
+ if (!filePath || !fs.existsSync(filePath)) return {};
16
+
17
+ const extension = path.extname(filePath).toLowerCase();
18
+ try {
19
+ if (extension === '.pdf') {
20
+ const bytes = await fs.promises.readFile(filePath);
21
+ const pdf = await PDFDocument.load(bytes, {
22
+ ignoreEncryption: true,
23
+ updateMetadata: false,
24
+ });
25
+ const pageCount = pdf.getPageCount();
26
+ return pageCount > 0 ? { pageCount, pageCountSource: 'pdf' } : {};
27
+ }
28
+
29
+ if (extension === '.docx') {
30
+ const archive = new AdmZip(filePath);
31
+ const appProperties = archive.getEntry('docProps/app.xml');
32
+ if (!appProperties) return {};
33
+ const xml = appProperties.getData().toString('utf-8');
34
+ const match = xml.match(/<(?:\w+:)?Pages>\s*(\d+)\s*<\/(?:\w+:)?Pages>/i);
35
+ const pageCount = match ? Number(match[1]) : 0;
36
+ return pageCount > 0
37
+ ? { pageCount, pageCountSource: 'docx_metadata' }
38
+ : {};
39
+ }
40
+ } catch (error) {
41
+ logger.warn('Could not read local document page count', {
42
+ filePath: path.basename(filePath),
43
+ error: error instanceof Error ? error.message : String(error),
44
+ });
45
+ }
46
+
47
+ return {};
48
+ }