nexusbert's picture
push new changes
6c9bf94
import express from "express";
import multer from "multer";
import { classifyFashionImage, identifyFashionItem } from "../utils/hfClient";
import { AppDataSource } from "../utils/dataSource";
import { WardrobeItem } from "../entity/WardrobeItem";
import { authenticateToken, AuthRequest } from "../middleware/auth";
import { normalizeCategory } from "../utils/categoryNormalizer";
import { generate3DModel, convert3DModelToBase64 } from "../utils/tencent3D";
import { readyPlayerMeClient } from "../utils/readyPlayerMe";
import { mapCategoryToAssetType, getAssetGender } from "../utils/assetTypeMapper";
const router = express.Router();
const upload = multer({
storage: multer.memoryStorage(),
limits: {
fileSize: 10 * 1024 * 1024,
},
fileFilter: (req, file, cb) => {
if (file.mimetype.startsWith("image/")) {
cb(null, true);
} else {
cb(new Error("Only image files are allowed"));
}
},
});
router.post("/", authenticateToken, upload.array("image", 20), async (req: AuthRequest, res) => {
try {
const userId = req.userId!;
const files = req.files as Express.Multer.File[];
const selectedStyle = req.body.style || "casual";
if (!files || files.length === 0) {
return res.status(400).json({
success: false,
error: "At least one image file is required. Please upload 1-20 image files."
});
}
if (files.length > 20) {
return res.status(400).json({
success: false,
error: "Maximum 20 images allowed per upload."
});
}
const itemRepo = AppDataSource.getRepository(WardrobeItem);
const uploadedItems: WardrobeItem[] = [];
const failedFiles: string[] = [];
for (let i = 0; i < files.length; i++) {
const file = files[i];
try {
console.log(`Processing image ${i + 1}/${files.length}: ${file.originalname}`);
const imageForClassification = file.buffer;
const imageMimeType = file.mimetype;
let color: string | null = null;
let itemLabel: string = "unknown";
let normalizedCategory: string = "unknown";
try {
const identifyResult = await identifyFashionItem(
imageForClassification,
file.originalname,
imageMimeType,
5
);
if (identifyResult) {
if (identifyResult.detected_components) {
const { top_colors, top_items } = identifyResult.detected_components;
if (top_colors && top_colors.length > 0 && top_colors[0].score > 0.2) {
color = top_colors[0].color;
}
if (top_items && top_items.length > 0) {
itemLabel = top_items[0].item;
normalizedCategory = normalizeCategory(itemLabel);
}
}
const itemFromPrecise = identifyResult.precise_identification?.find(
(item) => item.type === "item"
);
if (itemFromPrecise && itemFromPrecise.score > 0.3) {
itemLabel = itemFromPrecise.label;
normalizedCategory = normalizeCategory(itemLabel);
}
if (identifyResult.detected_components?.top_colors && identifyResult.detected_components.top_colors.length > 0) {
const topColor = identifyResult.detected_components.top_colors[0];
if (topColor.score > 0.2 && !color) {
color = topColor.color;
}
}
console.log(`Identification: ${itemLabel} | Color: ${color || "none"} | Category: ${normalizedCategory}`);
}
} catch (identifyError: any) {
console.warn(`⚠️ Identification failed for ${file.originalname}:`, identifyError.message);
try {
const classificationResults = await classifyFashionImage(
imageForClassification,
file.originalname,
imageMimeType
);
if (classificationResults.length > 0) {
itemLabel = classificationResults[0].label;
normalizedCategory = normalizeCategory(itemLabel);
}
} catch (classifyError: any) {
console.warn(`⚠️ Classification also failed for ${file.originalname}:`, classifyError.message);
}
}
const nameParts: string[] = [];
if (color) nameParts.push(color);
nameParts.push(itemLabel);
const itemName = nameParts.join(" ");
const base64Image = `data:${file.mimetype};base64,${file.buffer.toString("base64")}`;
console.log(`βœ… Image converted to base64: ${itemName}`);
let model3dUrl: string | undefined = undefined;
try {
console.log(`πŸ”„ Generating 3D model for: ${file.originalname}`);
const threeDResult = await generate3DModel(
file.buffer,
file.originalname,
file.mimetype
);
if (threeDResult && threeDResult.modelFile) {
const base64Model = await convert3DModelToBase64(threeDResult.modelFile);
if (base64Model) {
model3dUrl = base64Model;
console.log(`βœ… 3D model generated and converted to base64 for: ${file.originalname}`);
} else {
console.warn(`⚠️ Failed to convert 3D model to base64 for: ${file.originalname}`);
}
} else {
console.warn(`⚠️ 3D model generation returned no file for: ${file.originalname}`);
}
} catch (threeDError: any) {
console.warn(`⚠️ 3D model generation failed for ${file.originalname}:`, threeDError.message);
}
const style = selectedStyle;
const newItem = itemRepo.create({
imageUrl: base64Image,
processedImageUrl: undefined,
model3dUrl: model3dUrl,
category: normalizedCategory,
style: style,
name: itemName,
brand: undefined,
color: color || undefined,
userId: userId,
});
const savedItem = await itemRepo.save(newItem);
if (model3dUrl && process.env.READY_PLAYER_ME_APPLICATION_ID && process.env.READY_PLAYER_ME_ORGANIZATION_ID) {
try {
console.log(`πŸ”„ Creating Ready Player Me asset for: ${itemName} (category: ${normalizedCategory})`);
const assetType = mapCategoryToAssetType(normalizedCategory);
if (assetType) {
console.log(` - Asset type mapped: ${assetType}`);
// Upload 3D model file
const base64Data = model3dUrl.split(',')[1];
const binaryString = Buffer.from(base64Data, 'base64');
console.log(` - Uploading 3D model file (${binaryString.length} bytes)...`);
const modelUrl = await readyPlayerMeClient.uploadAssetFile(
binaryString,
`${itemName.replace(/\s+/g, '-')}.glb`,
"model/gltf-binary"
);
if (modelUrl) {
console.log(` - 3D model uploaded: ${modelUrl}`);
// Upload icon image file
const iconBase64Data = base64Image.split(',')[1];
const iconMimeType = file.mimetype;
const iconBuffer = Buffer.from(iconBase64Data, 'base64');
console.log(` - Uploading icon image (${iconBuffer.length} bytes, ${iconMimeType})...`);
const iconExtension = iconMimeType === 'image/png' ? 'png' : iconMimeType === 'image/jpeg' ? 'jpg' : 'jpg';
const iconUrl = await readyPlayerMeClient.uploadAssetFile(
iconBuffer,
`${itemName.replace(/\s+/g, '-')}-icon.${iconExtension}`,
iconMimeType
);
if (iconUrl) {
console.log(` - Icon uploaded: ${iconUrl}`);
console.log(` - Creating asset with type: ${assetType}, gender: ${getAssetGender()}`);
console.log(` - Asset data:`, {
name: itemName,
type: assetType,
gender: getAssetGender(),
modelUrl: modelUrl.substring(0, 100) + '...',
iconUrl: iconUrl.substring(0, 100) + '...',
organizationId: process.env.READY_PLAYER_ME_ORGANIZATION_ID?.substring(0, 10) + '...',
applicationId: process.env.READY_PLAYER_ME_APPLICATION_ID?.substring(0, 10) + '...',
});
const asset = await readyPlayerMeClient.createAsset({
name: itemName,
type: assetType,
gender: getAssetGender(),
modelUrl: modelUrl,
iconUrl: iconUrl,
organizationId: process.env.READY_PLAYER_ME_ORGANIZATION_ID!,
locked: false,
applications: [{
id: process.env.READY_PLAYER_ME_APPLICATION_ID!,
organizationId: process.env.READY_PLAYER_ME_ORGANIZATION_ID!,
isVisibleInEditor: true,
}],
});
if (asset) {
savedItem.readyPlayerMeAssetId = asset.id;
await itemRepo.save(savedItem);
console.log(`βœ… Created Ready Player Me asset: ${asset.id} for item ${savedItem.id}`);
} else {
console.error(`❌ Failed to create Ready Player Me asset: createAsset returned null`);
console.error(` This usually means the API call failed. Check READY_API_KEY permissions.`);
}
} else {
console.error(`❌ Failed to upload icon image to Ready Player Me temporary storage`);
console.error(` Check if READY_API_KEY has permission to upload files.`);
}
} else {
console.error(`❌ Failed to upload 3D model to Ready Player Me temporary storage`);
}
} else {
console.warn(`⚠️ Could not map category "${normalizedCategory}" to a Ready Player Me asset type`);
}
} catch (assetError: any) {
console.error(`❌ Error creating Ready Player Me asset for ${file.originalname}:`, assetError.message);
if (assetError.response) {
console.error(` Response status: ${assetError.response.status}`);
console.error(` Response data:`, JSON.stringify(assetError.response.data, null, 2));
}
if (assetError.stack) {
console.error(` Stack trace:`, assetError.stack);
}
}
} else {
if (!model3dUrl) {
console.warn(`⚠️ Skipping Ready Player Me asset creation: no 3D model generated`);
} else if (!process.env.READY_PLAYER_ME_APPLICATION_ID) {
console.warn(`⚠️ Skipping Ready Player Me asset creation: READY_PLAYER_ME_APPLICATION_ID not set`);
} else if (!process.env.READY_PLAYER_ME_ORGANIZATION_ID) {
console.warn(`⚠️ Skipping Ready Player Me asset creation: READY_PLAYER_ME_ORGANIZATION_ID not set`);
}
}
uploadedItems.push(savedItem);
console.log(`Successfully processed: ${file.originalname} -> ${itemName} (${normalizedCategory})`);
} catch (error: any) {
console.error(`❌ Error processing file ${file.originalname}:`, error);
failedFiles.push(file.originalname);
}
}
if (uploadedItems.length === 0) {
return res.status(500).json({
success: false,
error: "Failed to upload any images. Please try again."
});
}
res.json({
success: true,
items: uploadedItems,
count: uploadedItems.length,
failed: failedFiles.length,
failedFiles: failedFiles.length > 0 ? failedFiles : undefined
});
} catch (error: any) {
console.error("Upload error:", error);
res.status(500).json({
success: false,
error: error.message || "Upload failed"
});
}
});
export default router;