nexusbert's picture
ready player implementation
9557505
import { Client } from "@gradio/client";
import dotenv from "dotenv";
dotenv.config();
const TENCENT_3D_SPACE = "tencent/Hunyuan3D-2.1";
const HF_TOKEN = process.env.HF_TOKEN || process.env.hface;
export interface Tencent3DResult {
modelFile: Blob | File | Buffer | null;
htmlOutput: string;
meshStats: any;
seed: number;
}
export async function generate3DModel(
imageBuffer: Buffer,
filename: string,
mimetype: string
): Promise<Tencent3DResult | null> {
try {
console.log(`Connecting to Tencent 3D API for: ${filename}`);
const connectOptions: any = {};
if (HF_TOKEN) {
connectOptions.hf_token = HF_TOKEN;
console.log(` Using HF_TOKEN for authentication`);
} else {
console.warn(` No HF_TOKEN found in environment variables (checked HF_TOKEN and hface)`);
}
const client = await Client.connect(TENCENT_3D_SPACE, connectOptions);
const imageBlob = new Blob([imageBuffer], { type: mimetype });
console.log(`Generating 3D model for: ${filename}`);
const result = await client.predict("/shape_generation", {
image: imageBlob,
mv_image_front: imageBlob,
mv_image_back: imageBlob,
mv_image_left: imageBlob,
mv_image_right: imageBlob,
steps: 30,
guidance_scale: 5,
seed: 1234,
octree_resolution: 256,
check_box_rembg: true,
num_chunks: 8000,
randomize_seed: true,
});
console.log(`3D model generated successfully for: ${filename}`);
const data = result.data as any[];
return {
modelFile: data[0] || null,
htmlOutput: data[1] || "",
meshStats: data[2] || null,
seed: data[3] || 1234,
};
} catch (error: any) {
console.error(`Error generating 3D model for ${filename}:`, error.message);
return null;
}
}
export async function convert3DModelToBase64(
modelFile: Blob | File | Buffer | any
): Promise<string | null> {
try {
let buffer: Buffer;
if (modelFile instanceof Buffer) {
buffer = modelFile;
} else if (modelFile instanceof Blob) {
const arrayBuffer = await modelFile.arrayBuffer();
buffer = Buffer.from(arrayBuffer);
} else if (modelFile instanceof File) {
const arrayBuffer = await modelFile.arrayBuffer();
buffer = Buffer.from(arrayBuffer);
} else if (modelFile && typeof modelFile === 'object') {
// Handle Gradio client file objects
if (typeof modelFile.arrayBuffer === 'function') {
const arrayBuffer = await modelFile.arrayBuffer();
buffer = Buffer.from(arrayBuffer);
} else if (modelFile.value) {
// Gradio file component returns { value: url or object, __type__: "file" }
let fileUrl: string;
if (typeof modelFile.value === 'string') {
fileUrl = modelFile.value;
} else if (modelFile.value && typeof modelFile.value === 'object' && modelFile.value.url) {
fileUrl = modelFile.value.url;
} else if (modelFile.value && typeof modelFile.value === 'object' && modelFile.value.path) {
// Server-side file path
const fs = require('fs');
buffer = fs.readFileSync(modelFile.value.path);
const base64 = buffer.toString("base64");
return `data:model/gltf-binary;base64,${base64}`;
} else {
console.error("Unsupported file value format:", typeof modelFile.value, modelFile.value);
return null;
}
try {
const response = await fetch(fileUrl);
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const arrayBuffer = await response.arrayBuffer();
buffer = Buffer.from(arrayBuffer);
} catch (fetchError: any) {
console.error("Error fetching 3D model from URL:", fetchError.message);
return null;
}
} else if (modelFile.url) {
// Gradio returns file objects with URL - fetch the file
try {
const response = await fetch(modelFile.url);
const arrayBuffer = await response.arrayBuffer();
buffer = Buffer.from(arrayBuffer);
} catch (fetchError: any) {
console.error("Error fetching 3D model from URL:", fetchError.message);
return null;
}
} else if (modelFile.data) {
// Handle if it's wrapped in a data property
if (Buffer.isBuffer(modelFile.data)) {
buffer = modelFile.data;
} else if (typeof modelFile.data === 'string') {
// If it's already a base64 string or URL
if (modelFile.data.startsWith('data:')) {
return modelFile.data;
}
buffer = Buffer.from(modelFile.data, 'base64');
} else {
buffer = Buffer.from(modelFile.data);
}
} else if (modelFile.path) {
// If it's a file path (server-side), read it
const fs = require('fs');
buffer = fs.readFileSync(modelFile.path);
} else {
// Try to convert the object to buffer
try {
buffer = Buffer.from(modelFile);
} catch {
console.error("Unsupported file type for 3D model conversion:", typeof modelFile, modelFile.constructor?.name);
return null;
}
}
} else {
console.error("Unsupported file type for 3D model conversion:", typeof modelFile);
return null;
}
const base64 = buffer.toString("base64");
return `data:model/gltf-binary;base64,${base64}`;
} catch (error: any) {
console.error("Error converting 3D model to base64:", error.message);
return null;
}
}