sharktide's picture
download
raw
4.62 kB
// Actual JS SDK v3 /s3/ integration. Credentials stay in memory and are revoked.
import { readFileSync } from "node:fs";
import https from "node:https";
import { randomUUID } from "node:crypto";
import {
S3Client, CreateBucketCommand, PutObjectCommand, GetObjectCommand, ListObjectsV2Command,
CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand,
AbortMultipartUploadCommand, DeleteObjectCommand, DeleteBucketCommand,
} from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
const root = new URL("../../work/buckets-sandbox/", import.meta.url);
const config = JSON.parse(readFileSync(new URL("config.json", root)));
const tokens = JSON.parse(readFileSync(new URL("credentials.json", root)));
let endpoint = process.env.BUCKETS_E2E_ENDPOINT || config.BUCKETS_PUBLIC_ENDPOINT;
const base = endpoint.replace(/\/s3\/$/, "");
async function native(path, method = "GET", value) {
const response = await fetch(base + path, { method, headers: {
Authorization: "Bearer " + tokens.user1_personal, "Content-Type": "application/json",
}, body: value === undefined ? undefined : JSON.stringify(value) });
if (!response.ok) throw new Error("Native API failed: " + response.status);
return response.status === 204 ? null : response.json();
}
const credential = await native("/v1/buckets/s3-credentials", "POST", { permissions: ["storage.read", "storage.write", "storage.admin"], rate_limit_rpm: 600 });
endpoint = new URL(endpoint).origin + new URL(credential.endpoint).pathname;
const client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true, maxAttempts: 1,
credentials: { accessKeyId: credential.access_key_id, secretAccessKey: credential.secret_access_key } });
client.middlewareStack.add((next, context) => async (args) => { try { return await next(args); } catch (error) { console.error("Failed SDK operation:", context.commandName); throw error; } }, { step: "initialize", name: "testOperationDiagnostic" });
const Bucket = "js-e2e-" + randomUUID().slice(0, 12);
const Key = "models/模型//../literal%2Fkey";
try {
await client.send(new CreateBucketCommand({ Bucket }));
await client.send(new PutObjectCommand({ Bucket, Key, Body: Buffer.from("JS SDK content"), ContentType: "text/plain" }));
const response = await client.send(new GetObjectCommand({ Bucket, Key }));
if (await response.Body.transformToString() !== "JS SDK content") throw new Error("Object content mismatch");
const listed = await client.send(new ListObjectsV2Command({ Bucket, Prefix: "models/" }));
if (listed.KeyCount !== 1) throw new Error("List mismatch");
const url = await getSignedUrl(client, new GetObjectCommand({ Bucket, Key }), { expiresIn: 60 });
// WHATWG fetch normalizes literal dot segments before transmission. Use the
// original request target for that key, just as the SDK's HTTP handler does.
const rawTarget = url.slice(url.indexOf("/", "https://".length));
const signedContent = await new Promise((resolve, reject) => {
https.get({ hostname: "localhost", port: Number(new URL(endpoint).port), path: rawTarget,
ca: readFileSync(new URL("localhost.pem", root)) }, (response) => {
const buffers = []; response.on("data", (chunk) => buffers.push(chunk));
response.on("end", () => response.statusCode === 200 ? resolve(Buffer.concat(buffers).toString()) : reject(new Error("Presign HTTP " + response.statusCode)));
}).on("error", reject);
});
if (signedContent !== "JS SDK content") throw new Error("Presign mismatch");
const upload = await client.send(new CreateMultipartUploadCommand({ Bucket, Key: "multipart" }));
const part = await client.send(new UploadPartCommand({ Bucket, Key: "multipart", UploadId: upload.UploadId, PartNumber: 1, Body: Buffer.alloc(5 * 1024 ** 2, 65) }));
await client.send(new CompleteMultipartUploadCommand({ Bucket, Key: "multipart", UploadId: upload.UploadId,
MultipartUpload: { Parts: [{ PartNumber: 1, ETag: part.ETag }] } }));
const abort = await client.send(new CreateMultipartUploadCommand({ Bucket, Key: "abort" }));
await client.send(new AbortMultipartUploadCommand({ Bucket, Key: "abort", UploadId: abort.UploadId }));
await client.send(new DeleteObjectCommand({ Bucket, Key }));
await client.send(new DeleteObjectCommand({ Bucket, Key: "multipart" }));
await client.send(new DeleteBucketCommand({ Bucket }));
console.log("PASS JS SDK v3: signed /s3/ paths, encoded keys, checksums, presign, multipart, abort, cleanup");
} finally {
await native("/v1/buckets/s3-credentials/" + credential.id, "DELETE");
client.destroy();
}

Xet Storage Details

Size:
4.62 kB
·
Xet hash:
c9357b348446121d4c7fad2726c80f4ae90040426878d0ee69412a630cd7122b

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.