File size: 34,960 Bytes
d2ec5e7 96c4f85 d2ec5e7 96c4f85 d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 96c4f85 d67e1e8 124fc9e d67e1e8 96c4f85 d2ec5e7 e097ca3 124fc9e e097ca3 d2ec5e7 124fc9e d2ec5e7 96c4f85 124fc9e 96c4f85 d2ec5e7 96c4f85 e097ca3 d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 942ff10 e097ca3 942ff10 e097ca3 942ff10 96c4f85 124fc9e 96c4f85 124fc9e 96c4f85 124fc9e 96c4f85 d2ec5e7 96c4f85 d2ec5e7 96c4f85 d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 e097ca3 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 e097ca3 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 942ff10 d2ec5e7 124fc9e 96c4f85 d2ec5e7 124fc9e d2ec5e7 124fc9e 96c4f85 d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 124fc9e d2ec5e7 942ff10 d2ec5e7 942ff10 124fc9e 942ff10 124fc9e 942ff10 d2ec5e7 942ff10 96c4f85 942ff10 96c4f85 942ff10 d2ec5e7 942ff10 d2ec5e7 124fc9e 942ff10 e097ca3 124fc9e e097ca3 d2ec5e7 124fc9e 96c4f85 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 | import _ from "lodash";
import fs from "fs-extra";
import axios from "axios";
import APIException from "@/lib/exceptions/APIException.ts";
import EX from "@/api/consts/exceptions.ts";
import util from "@/lib/util.ts";
import { getCredit, receiveCredit, request, parseRegionFromToken, getAssistantId, checkImageContent, RegionInfo } from "./core.ts";
import logger from "@/lib/logger.ts";
import { SmartPoller, PollingStatus } from "@/lib/smart-poller.ts";
import { DEFAULT_ASSISTANT_ID_CN, DEFAULT_ASSISTANT_ID_US, DEFAULT_ASSISTANT_ID_HK, DEFAULT_ASSISTANT_ID_JP, DEFAULT_ASSISTANT_ID_SG, DEFAULT_VIDEO_MODEL, DRAFT_VERSION, DRAFT_VERSION_OMNI, OMNI_BENEFIT_TYPE, OMNI_BENEFIT_TYPE_FAST, VIDEO_MODEL_MAP, VIDEO_MODEL_MAP_US, VIDEO_MODEL_MAP_ASIA } from "@/api/consts/common.ts";
import { uploadImageBuffer, ImageUploadResult } from "@/lib/image-uploader.ts";
import { uploadVideoBuffer, VideoUploadResult } from "@/lib/video-uploader.ts";
import { extractVideoUrl, fetchHighQualityVideoUrl } from "@/lib/image-utils.ts";
import { uploadVideoFromUrl } from "@/lib/video-uploader.ts";
export const DEFAULT_MODEL = DEFAULT_VIDEO_MODEL;
export function getModel(model: string, regionInfo: RegionInfo) {
// 根据站点选择不同的模型映射
let modelMap: Record<string, string>;
if (regionInfo.isUS) {
modelMap = VIDEO_MODEL_MAP_US;
} else if (regionInfo.isHK || regionInfo.isJP || regionInfo.isSG) {
modelMap = VIDEO_MODEL_MAP_ASIA;
} else {
modelMap = VIDEO_MODEL_MAP;
}
return modelMap[model] || modelMap[DEFAULT_MODEL] || VIDEO_MODEL_MAP[DEFAULT_MODEL];
}
function getVideoBenefitType(model: string): string {
// veo3.1 模型 (需先于 veo3 检查)
if (model.includes("veo3.1")) {
return "generate_video_veo3.1";
}
// veo3 模型
if (model.includes("veo3")) {
return "generate_video_veo3";
}
// sora2 模型
if (model.includes("sora2")) {
return "generate_video_sora2";
}
if (model.includes("40_pro")) {
return "dreamina_video_seedance_20_pro";
}
if (model.includes("40")) {
return "dreamina_video_seedance_20_fast";
}
if (model.includes("3.5_pro")) {
return "dreamina_video_seedance_15_pro";
}
if (model.includes("3.5")) {
return "dreamina_video_seedance_15";
}
return "basic_video_operation_vgfm_v_three";
}
// 处理本地上传的文件
async function uploadImageFromFile(file: any, refreshToken: string, regionInfo: RegionInfo): Promise<ImageUploadResult> {
try {
logger.info(`开始从本地文件上传视频图片: ${file.originalFilename} (路径: ${file.filepath})`);
const imageBuffer = await fs.readFile(file.filepath);
return await uploadImageBuffer(imageBuffer, refreshToken, regionInfo);
} catch (error: any) {
logger.error(`从本地文件上传视频图片失败: ${error.message}`);
throw error;
}
}
// 处理来自URL的图片
async function uploadImageFromUrl(imageUrl: string, refreshToken: string, regionInfo: RegionInfo): Promise<ImageUploadResult> {
try {
logger.info(`开始从URL下载并上传视频图片: ${imageUrl}`);
const imageResponse = await axios.get(imageUrl, {
responseType: 'arraybuffer',
proxy: false,
});
if (imageResponse.status < 200 || imageResponse.status >= 300) {
throw new Error(`下载图片失败: ${imageResponse.status}`);
}
const imageBuffer = imageResponse.data;
return await uploadImageBuffer(imageBuffer, refreshToken, regionInfo);
} catch (error: any) {
logger.error(`从URL上传视频图片失败: ${error.message}`);
throw error;
}
}
/**
* 解析 omni_reference 模式的 prompt,将 @引用 拆解为 meta_list
* 输入: "@image_file_1作为首帧,@image_file_2作为尾帧,运动动作模仿@video_file"
* 输出: 交替的 text + material_ref 段
*/
function parseOmniPrompt(prompt: string, materialRegistry: Map<string, any>): any[] {
// 收集所有可识别的引用名(字段名 + 原始文件名),转义正则特殊字符
const refNames = [...materialRegistry.keys()]
.sort((a, b) => b.length - a.length) // 长名优先匹配
.map(n => n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
if (refNames.length === 0) {
return [{ meta_type: "text", text: prompt }];
}
const pattern = new RegExp(`@(${refNames.join('|')})`, 'g');
const meta_list: any[] = [];
let lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = pattern.exec(prompt)) !== null) {
// 文本段
if (match.index > lastIndex) {
const textSegment = prompt.slice(lastIndex, match.index);
if (textSegment) {
meta_list.push({ meta_type: "text", text: textSegment });
}
}
// 引用段
const refName = match[1];
const entry = materialRegistry.get(refName);
if (entry) {
meta_list.push({
meta_type: entry.type,
text: "",
material_ref: { material_idx: entry.idx },
});
}
lastIndex = pattern.lastIndex;
}
// 尾部文本
if (lastIndex < prompt.length) {
meta_list.push({ meta_type: "text", text: prompt.slice(lastIndex) });
}
// 如果没有任何 @ 引用,把整个 prompt 作为文本段
if (meta_list.length === 0) {
meta_list.push({ meta_type: "text", text: prompt });
}
return meta_list;
}
/**
* 生成视频
*
* @param _model 模型名称
* @param prompt 提示词
* @param options 选项
* @param refreshToken 刷新令牌
* @returns 视频URL
*/
export async function generateVideo(
_model: string,
prompt: string,
{
ratio = "1:1",
resolution = "720p",
duration = 5,
filePaths = [],
files = {},
httpRequest,
functionMode = "first_last_frames",
}: {
ratio?: string;
resolution?: string;
duration?: number;
filePaths?: string[];
files?: any;
httpRequest?: any;
functionMode?: string;
},
refreshToken: string
) {
// 检测区域
const regionInfo = parseRegionFromToken(refreshToken);
const { isInternational } = regionInfo;
logger.info(`视频生成区域检测: isInternational=${isInternational}`);
const model = getModel(_model, regionInfo);
const isVeo3 = model.includes("veo3");
const isSora2 = model.includes("sora2");
const is35Pro = model.includes("3.5_pro");
const is40Pro = model.includes("40_pro");
const is40 = model.includes("40") && !model.includes("40_pro");
// 只有 video-3.0 和 video-3.0-fast 支持 resolution 参数(3.0-pro 和 3.5-pro 不支持)
const supportsResolution = (model.includes("vgfm_3.0") || model.includes("vgfm_3.0_fast")) && !model.includes("_pro");
// 将秒转换为毫秒
// veo3 模型固定 8 秒
// sora2 模型支持 4秒、8秒、12秒,默认4秒
// 3.5-pro 模型支持 5秒、10秒、12秒,默认5秒
// 4.0-pro (seedance 2.0) 和 4.0 (seedance 2.0-fast) 模型支持 4~15秒,默认5秒
// 其他模型支持 5秒、10秒,默认5秒
let durationMs: number;
let actualDuration: number;
if (isVeo3) {
durationMs = 8000;
actualDuration = 8;
} else if (isSora2) {
if (duration === 12) {
durationMs = 12000;
actualDuration = 12;
} else if (duration === 8) {
durationMs = 8000;
actualDuration = 8;
} else {
durationMs = 4000;
actualDuration = 4;
}
} else if (is40Pro || is40) {
// seedance 2.0 和 2.0-fast: 支持 4~15 秒,clamp 到有效范围,默认 5 秒
actualDuration = Math.max(4, Math.min(15, duration));
durationMs = actualDuration * 1000;
} else if (is35Pro) {
if (duration === 12) {
durationMs = 12000;
actualDuration = 12;
} else if (duration === 10) {
durationMs = 10000;
actualDuration = 10;
} else {
durationMs = 5000;
actualDuration = 5;
}
} else {
durationMs = duration === 10 ? 10000 : 5000;
actualDuration = duration === 10 ? 10 : 5;
}
logger.info(`使用模型: ${_model} 映射模型: ${model} 比例: ${ratio} 分辨率: ${supportsResolution ? resolution : '不支持'} 时长: ${actualDuration}s`);
// 检查积分
const { totalCredit } = await getCredit(refreshToken);
if (totalCredit <= 0) {
logger.info("积分为 0,尝试收取今日积分...");
try {
await receiveCredit(refreshToken);
} catch (receiveError) {
logger.warn(`收取积分失败: ${receiveError.message}. 这可能是因为: 1) 今日已收取过积分, 2) 账户受到风控限制, 3) 需要在官网手动收取首次积分`);
throw new APIException(EX.API_VIDEO_GENERATION_FAILED,
`积分不足且无法自动收取。请访问即梦官网手动收取首次积分,或检查账户状态。`);
}
}
const isOmniMode = functionMode === "omni_reference";
// omni_reference 仅支持 seedance 2.0 (40_pro) 和 2.0-fast (40) 模型
if (isOmniMode && !is40Pro && !is40) {
throw new APIException(EX.API_REQUEST_FAILED,
`omni_reference 模式仅支持 jimeng-video-seedance-2.0 和 jimeng-video-seedance-2.0-fast 模型`);
}
let requestData: any;
if (isOmniMode) {
// ========== omni_reference 分支 ==========
logger.info(`进入 omni_reference 全能模式`);
// 素材注册表: fieldName → { idx, type, uploadResult }
interface MaterialEntry {
idx: number;
type: "image" | "video";
fieldName: string;
originalFilename: string;
imageUri?: string;
imageWidth?: number;
imageHeight?: number;
imageFormat?: string;
videoResult?: VideoUploadResult;
}
const materialRegistry: Map<string, MaterialEntry> = new Map();
let materialIdx = 0;
// canonical key 集合,防止 originalFilename 覆盖
const canonicalKeys = new Set<string>();
canonicalKeys.add('image_file');
canonicalKeys.add('video_file');
for (let i = 1; i <= 9; i++) canonicalKeys.add(`image_file_${i}`);
for (let i = 1; i <= 3; i++) canonicalKeys.add(`video_file_${i}`);
// 安全注册别名:originalFilename 不与 canonical key 冲突时才注册
function registerAlias(filename: string, entry: MaterialEntry) {
if (!canonicalKeys.has(filename) && !materialRegistry.has(filename)) {
materialRegistry.set(filename, entry);
}
}
// 收集所有需要处理的图片和视频字段
const imageFields: string[] = [];
const videoFields: string[] = [];
// 检测上传的文件
if (files) {
for (const fieldName of Object.keys(files)) {
if (fieldName === 'image_file' || fieldName.startsWith('image_file_')) imageFields.push(fieldName);
else if (fieldName === 'video_file' || fieldName.startsWith('video_file_')) videoFields.push(fieldName);
}
}
// 检测URL字段
for (let i = 1; i <= 9; i++) {
const fieldName = `image_file_${i}`;
if (typeof httpRequest?.body?.[fieldName] === 'string' && httpRequest.body[fieldName].startsWith('http')) {
if (!imageFields.includes(fieldName)) imageFields.push(fieldName);
}
}
for (let i = 1; i <= 3; i++) {
const fieldName = `video_file_${i}`;
if (typeof httpRequest?.body?.[fieldName] === 'string' && httpRequest.body[fieldName].startsWith('http')) {
if (!videoFields.includes(fieldName)) videoFields.push(fieldName);
}
}
// 检测不带数字后缀的裸名 URL 字段
if (typeof httpRequest?.body?.image_file === 'string' && httpRequest.body.image_file.startsWith('http')) {
if (!imageFields.includes('image_file')) imageFields.push('image_file');
}
if (typeof httpRequest?.body?.video_file === 'string' && httpRequest.body.video_file.startsWith('http')) {
if (!videoFields.includes('video_file')) videoFields.push('video_file');
}
// 检查是否有素材
const hasFilePaths = filePaths && filePaths.length > 0;
if (imageFields.length === 0 && videoFields.length === 0 && !hasFilePaths) {
throw new APIException(EX.API_REQUEST_FAILED,
`omni_reference 模式需要至少上传一个素材文件 (image_file_*, video_file_*) 或提供素材URL`);
}
let totalVideoDuration = 0; // 累计视频时长
// 串行上传图片素材
for (const fieldName of imageFields) {
const imageFile = files?.[fieldName];
const imageUrlField = httpRequest?.body?.[fieldName];
try {
logger.info(`[omni] 上传 ${fieldName}`);
let imgResult: ImageUploadResult;
if (imageFile) {
// 本地文件上传
const buf = await fs.readFile(imageFile.filepath);
imgResult = await uploadImageBuffer(buf, refreshToken, regionInfo);
await checkImageContent(imgResult.uri, refreshToken, regionInfo);
const entry: MaterialEntry = {
idx: materialIdx++,
type: "image",
fieldName,
originalFilename: imageFile.originalFilename,
imageUri: imgResult.uri,
imageWidth: imgResult.width,
imageHeight: imgResult.height,
imageFormat: imgResult.format,
};
materialRegistry.set(fieldName, entry);
registerAlias(imageFile.originalFilename, entry);
logger.info(`[omni] ${fieldName} 上传成功: ${imgResult.uri} (${imgResult.width}x${imgResult.height})`);
} else if (imageUrlField && typeof imageUrlField === 'string' && imageUrlField.startsWith('http')) {
// URL上传
imgResult = await uploadImageFromUrl(imageUrlField, refreshToken, regionInfo);
await checkImageContent(imgResult.uri, refreshToken, regionInfo);
const entry: MaterialEntry = {
idx: materialIdx++,
type: "image",
fieldName,
originalFilename: imageUrlField,
imageUri: imgResult.uri,
imageWidth: imgResult.width,
imageHeight: imgResult.height,
imageFormat: imgResult.format,
};
materialRegistry.set(fieldName, entry);
logger.info(`[omni] ${fieldName} URL上传成功: ${imgResult.uri} (${imgResult.width}x${imgResult.height})`);
}
} catch (error: any) {
throw new APIException(EX.API_REQUEST_FAILED, `${fieldName} 处理失败: ${error.message}`);
}
}
// 通过 filePaths 数组补充未被占用的图片槽位
if (filePaths && filePaths.length > 0) {
let slotIndex = 1;
for (const url of filePaths) {
// 找到第一个未被占用的槽位
while (slotIndex <= 9 && materialRegistry.has(`image_file_${slotIndex}`)) {
slotIndex++;
}
if (slotIndex > 9) break; // 已达到最大数量
const fieldName = `image_file_${slotIndex}`;
try {
logger.info(`[omni] 从URL上传 ${fieldName}: ${url}`);
const imgResult = await uploadImageFromUrl(url, refreshToken, regionInfo);
await checkImageContent(imgResult.uri, refreshToken, regionInfo);
const entry: MaterialEntry = {
idx: materialIdx++,
type: "image",
fieldName,
originalFilename: url,
imageUri: imgResult.uri,
imageWidth: imgResult.width,
imageHeight: imgResult.height,
imageFormat: imgResult.format,
};
materialRegistry.set(fieldName, entry);
logger.info(`[omni] ${fieldName} URL上传成功: ${imgResult.uri} (${imgResult.width}x${imgResult.height})`);
} catch (error: any) {
throw new APIException(EX.API_REQUEST_FAILED, `${fieldName} URL图片处理失败: ${error.message}`);
}
slotIndex++;
}
}
// 串行上传视频素材
for (const fieldName of videoFields) {
const videoFile = files?.[fieldName];
const videoUrlField = httpRequest?.body?.[fieldName];
try {
logger.info(`[omni] 上传 ${fieldName}`);
let vResult: VideoUploadResult;
if (videoFile) {
// 本地文件上传
const buf = await fs.readFile(videoFile.filepath);
vResult = await uploadVideoBuffer(buf, refreshToken, regionInfo);
totalVideoDuration += vResult.videoMeta.duration;
const entry: MaterialEntry = {
idx: materialIdx++,
type: "video",
fieldName,
originalFilename: videoFile.originalFilename,
videoResult: vResult
};
materialRegistry.set(fieldName, entry);
registerAlias(videoFile.originalFilename, entry);
logger.info(`[omni] ${fieldName} 上传成功: vid=${vResult.vid}, ${vResult.videoMeta.width}x${vResult.videoMeta.height}, ${vResult.videoMeta.duration}s`);
} else if (videoUrlField && typeof videoUrlField === 'string' && videoUrlField.startsWith('http')) {
// URL上传
vResult = await uploadVideoFromUrl(videoUrlField, refreshToken, regionInfo);
totalVideoDuration += vResult.videoMeta.duration;
const entry: MaterialEntry = {
idx: materialIdx++,
type: "video",
fieldName,
originalFilename: videoUrlField,
videoResult: vResult
};
materialRegistry.set(fieldName, entry);
logger.info(`[omni] ${fieldName} URL上传成功: vid=${vResult.vid}, ${vResult.videoMeta.width}x${vResult.videoMeta.height}, ${vResult.videoMeta.duration}s`);
}
} catch (error: any) {
throw new APIException(EX.API_REQUEST_FAILED, `${fieldName} 处理失败: ${error.message}`);
}
}
// 验证视频总时长
const MAX_TOTAL_VIDEO_DURATION = 15;
if (!Number.isFinite(totalVideoDuration)) {
throw new APIException(EX.API_REQUEST_FAILED,
`视频时长数据异常,请检查视频文件`);
}
if (totalVideoDuration > MAX_TOTAL_VIDEO_DURATION) {
throw new APIException(EX.API_REQUEST_FAILED,
`视频总时长 ${totalVideoDuration.toFixed(2)}s 超过限制 (最大 ${MAX_TOTAL_VIDEO_DURATION}s)`);
}
logger.info(`[omni] 视频总时长: ${totalVideoDuration.toFixed(2)}s`);
// 构建 material_list(按注册顺序)
const orderedEntries = [...new Map([...materialRegistry].filter(([k, v]) => k === v.fieldName)).values()]
.sort((a, b) => a.idx - b.idx);
const material_list: any[] = [];
const materialTypes: number[] = [];
for (const entry of orderedEntries) {
if (entry.type === "image") {
material_list.push({
type: "",
id: util.uuid(),
material_type: "image",
image_info: {
type: "image",
id: util.uuid(),
source_from: "upload",
platform_type: 1,
name: "",
image_uri: entry.imageUri,
width: entry.imageWidth || 0,
height: entry.imageHeight || 0,
format: entry.imageFormat || "",
uri: entry.imageUri,
},
});
materialTypes.push(1);
} else {
const vm = entry.videoResult!;
material_list.push({
type: "",
id: util.uuid(),
material_type: "video",
video_info: {
type: "video",
id: util.uuid(),
source_from: "upload",
name: "",
vid: vm.vid,
fps: 0,
width: vm.videoMeta.width,
height: vm.videoMeta.height,
duration: Math.round(vm.videoMeta.duration * 1000),
},
});
materialTypes.push(2);
}
}
// 解析 prompt → meta_list
const meta_list = parseOmniPrompt(prompt, materialRegistry);
logger.info(`[omni] material_list: ${material_list.length} 项, meta_list: ${meta_list.length} 项, materialTypes: [${materialTypes}]`);
// 构建 omni payload
const componentId = util.uuid();
const submitId = util.uuid();
const sceneOption = {
type: "video",
scene: "BasicVideoGenerateButton",
modelReqKey: model,
videoDuration: actualDuration,
materialTypes,
reportParams: {
enterSource: "generate",
vipSource: "generate",
extraVipFunctionKey: model,
useVipFunctionDetailsReporterHoc: true,
},
};
const metricsExtra = JSON.stringify({
position: "page_bottom_box",
isDefaultSeed: 1,
originSubmitId: submitId,
isRegenerate: false,
enterFrom: "click",
functionMode: "omni_reference",
sceneOptions: JSON.stringify([sceneOption]),
});
// 根据模型选择 benefit_type
const omniBenefitType = is40 ? OMNI_BENEFIT_TYPE_FAST : OMNI_BENEFIT_TYPE;
requestData = {
params: {
aigc_features: "app_lip_sync",
web_version: "7.5.0",
da_version: DRAFT_VERSION_OMNI,
},
data: {
extend: {
root_model: model,
m_video_commerce_info: {
benefit_type: omniBenefitType,
resource_id: "generate_video",
resource_id_type: "str",
resource_sub_type: "aigc",
},
m_video_commerce_info_list: [{
benefit_type: omniBenefitType,
resource_id: "generate_video",
resource_id_type: "str",
resource_sub_type: "aigc",
}],
},
submit_id: submitId,
metrics_extra: metricsExtra,
draft_content: JSON.stringify({
type: "draft",
id: util.uuid(),
min_version: DRAFT_VERSION_OMNI,
min_features: ["AIGC_Video_UnifiedEdit"],
is_from_tsn: true,
version: DRAFT_VERSION_OMNI,
main_component_id: componentId,
component_list: [{
type: "video_base_component",
id: componentId,
min_version: "1.0.0",
aigc_mode: "workbench",
metadata: {
type: "",
id: util.uuid(),
created_platform: 3,
created_platform_version: "",
created_time_in_ms: Date.now().toString(),
created_did: "",
},
generate_type: "gen_video",
abilities: {
type: "",
id: util.uuid(),
gen_video: {
id: util.uuid(),
type: "",
text_to_video_params: {
type: "",
id: util.uuid(),
video_gen_inputs: [{
type: "",
id: util.uuid(),
min_version: DRAFT_VERSION_OMNI,
prompt: "",
video_mode: 2,
fps: 24,
duration_ms: durationMs,
unified_edit_input: {
type: "",
id: util.uuid(),
material_list,
meta_list,
},
idip_meta_list: [],
}],
video_aspect_ratio: ratio,
seed: Math.floor(Math.random() * 4294967296),
model_req_key: model,
priority: 0,
},
video_task_extra: metricsExtra,
},
},
process_type: 1,
}],
}),
http_common_info: {
aid: getAssistantId(regionInfo),
},
},
};
} else {
// ========== first_last_frames 分支(原有逻辑) ==========
let first_frame_image = undefined;
let end_frame_image = undefined;
let uploadIDs: string[] = [];
// 优先处理本地上传的文件
const uploadedFiles = _.values(files);
if (uploadedFiles && uploadedFiles.length > 0) {
logger.info(`检测到 ${uploadedFiles.length} 个本地上传文件,优先处理`);
for (let i = 0; i < uploadedFiles.length; i++) {
const file = uploadedFiles[i];
if (!file) continue;
try {
logger.info(`开始上传第 ${i + 1} 张本地图片: ${file.originalFilename}`);
const imgResult = await uploadImageFromFile(file, refreshToken, regionInfo);
if (imgResult) {
await checkImageContent(imgResult.uri, refreshToken, regionInfo);
uploadIDs.push(imgResult.uri);
logger.info(`第 ${i + 1} 张本地图片上传成功: ${imgResult.uri}`);
} else {
logger.error(`第 ${i + 1} 张本地图片上传失败: 未获取到 image_uri`);
}
} catch (error: any) {
logger.error(`第 ${i + 1} 张本地图片上传失败: ${error.message}`);
if (i === 0) {
throw new APIException(EX.API_REQUEST_FAILED, `首帧图片上传失败: ${error.message}`);
}
}
}
} else if (filePaths && filePaths.length > 0) {
logger.info(`未检测到本地上传文件,处理 ${filePaths.length} 个图片URL`);
for (let i = 0; i < filePaths.length; i++) {
const filePath = filePaths[i];
if (!filePath) {
logger.warn(`第 ${i + 1} 个图片URL为空,跳过`);
continue;
}
try {
logger.info(`开始上传第 ${i + 1} 个URL图片: ${filePath}`);
const imgResult = await uploadImageFromUrl(filePath, refreshToken, regionInfo);
if (imgResult) {
await checkImageContent(imgResult.uri, refreshToken, regionInfo);
uploadIDs.push(imgResult.uri);
logger.info(`第 ${i + 1} 个URL图片上传成功: ${imgResult.uri}`);
} else {
logger.error(`第 ${i + 1} 个URL图片上传失败: 未获取到 image_uri`);
}
} catch (error: any) {
logger.error(`第 ${i + 1} 个URL图片上传失败: ${error.message}`);
if (i === 0) {
throw new APIException(EX.API_REQUEST_FAILED, `首帧图片上传失败: ${error.message}`);
}
}
}
} else {
logger.info(`未提供图片文件或URL,将进行纯文本视频生成`);
}
if (uploadIDs.length > 0) {
logger.info(`图片上传完成,共成功 ${uploadIDs.length} 张`);
if (uploadIDs[0]) {
first_frame_image = {
format: "", height: 0, id: util.uuid(), image_uri: uploadIDs[0],
name: "", platform_type: 1, source_from: "upload", type: "image", uri: uploadIDs[0], width: 0,
};
logger.info(`设置首帧图片: ${uploadIDs[0]}`);
}
if (uploadIDs[1]) {
end_frame_image = {
format: "", height: 0, id: util.uuid(), image_uri: uploadIDs[1],
name: "", platform_type: 1, source_from: "upload", type: "image", uri: uploadIDs[1], width: 0,
};
logger.info(`设置尾帧图片: ${uploadIDs[1]}`);
}
}
const componentId = util.uuid();
const originSubmitId = util.uuid();
const flFunctionMode = "first_last_frames";
const sceneOption = {
type: "video",
scene: "BasicVideoGenerateButton",
...(supportsResolution ? { resolution } : {}),
modelReqKey: model,
videoDuration: actualDuration,
reportParams: {
enterSource: "generate",
vipSource: "generate",
extraVipFunctionKey: supportsResolution ? `${model}-${resolution}` : model,
useVipFunctionDetailsReporterHoc: true,
},
};
const metricsExtra = JSON.stringify({
promptSource: "custom",
isDefaultSeed: 1,
originSubmitId,
isRegenerate: false,
enterFrom: "click",
functionMode: flFunctionMode,
sceneOptions: JSON.stringify([sceneOption]),
});
const hasImageInput = uploadIDs.length > 0;
if (hasImageInput && ratio !== "1:1") {
logger.warn(`图生视频模式下,ratio参数将被忽略(由输入图片的实际比例决定),但resolution参数仍然有效`);
}
logger.info(`视频生成模式: ${uploadIDs.length}张图片 (首帧: ${!!first_frame_image}, 尾帧: ${!!end_frame_image}), resolution: ${resolution}`);
requestData = {
params: {
aigc_features: "app_lip_sync",
web_version: "7.5.0",
da_version: DRAFT_VERSION,
},
data: {
extend: {
root_model: model,
m_video_commerce_info: {
benefit_type: getVideoBenefitType(model),
resource_id: "generate_video",
resource_id_type: "str",
resource_sub_type: "aigc",
},
m_video_commerce_info_list: [{
benefit_type: getVideoBenefitType(model),
resource_id: "generate_video",
resource_id_type: "str",
resource_sub_type: "aigc",
}],
},
submit_id: util.uuid(),
metrics_extra: metricsExtra,
draft_content: JSON.stringify({
type: "draft",
id: util.uuid(),
min_version: "3.0.5",
min_features: [],
is_from_tsn: true,
version: DRAFT_VERSION,
main_component_id: componentId,
component_list: [{
type: "video_base_component",
id: componentId,
min_version: "1.0.0",
aigc_mode: "workbench",
metadata: {
type: "",
id: util.uuid(),
created_platform: 3,
created_platform_version: "",
created_time_in_ms: Date.now().toString(),
created_did: "",
},
generate_type: "gen_video",
abilities: {
type: "",
id: util.uuid(),
gen_video: {
id: util.uuid(),
type: "",
text_to_video_params: {
type: "",
id: util.uuid(),
video_gen_inputs: [{
type: "",
id: util.uuid(),
min_version: "3.0.5",
prompt,
video_mode: 2,
fps: 24,
duration_ms: durationMs,
...(supportsResolution ? { resolution } : {}),
first_frame_image,
end_frame_image,
idip_meta_list: [],
}],
video_aspect_ratio: ratio,
seed: Math.floor(Math.random() * 4294967296),
model_req_key: model,
priority: 0,
},
video_task_extra: metricsExtra,
},
},
process_type: 1,
}],
}),
http_common_info: {
aid: getAssistantId(regionInfo),
},
},
};
}
// 发送请求
const videoReferer = regionInfo.isCN
? "https://jimeng.jianying.com/ai-tool/generate?type=video"
: "https://dreamina.capcut.com/ai-tool/generate?type=video";
const { aigc_data } = await request(
"post",
"/mweb/v1/aigc_draft/generate",
refreshToken,
{
...requestData,
headers: { Referer: videoReferer },
}
);
const historyId = aigc_data.history_record_id;
if (!historyId)
throw new APIException(EX.API_IMAGE_GENERATION_FAILED, "记录ID不存在");
logger.info(`视频生成任务已提交,history_id: ${historyId},等待生成完成...`);
// 首次查询前等待,让服务器有时间处理请求
await new Promise((resolve) => setTimeout(resolve, 5000));
// 使用 SmartPoller 进行智能轮询
const maxPollCount = 900; // 增加轮询次数,支持更长的生成时间
let pollAttempts = 0;
const poller = new SmartPoller({
maxPollCount,
pollInterval: 20000, // 20秒基础间隔
expectedItemCount: 1,
type: 'video',
timeoutSeconds: 3600 // 60分钟超时
});
const { result: pollingResult, data: finalHistoryData } = await poller.poll(async () => {
pollAttempts++;
// 使用标准API请求方式
const result = await request("post", "/mweb/v1/get_history_by_ids", refreshToken, {
data: {
history_ids: [historyId],
},
});
// 检查响应中是否有该 history_id 的数据
// 由于 API 存在最终一致性,早期轮询可能暂时获取不到记录,返回处理中状态继续轮询
if (!result[historyId]) {
logger.warn(`API未返回历史记录 (轮询第${pollAttempts}次),historyId: ${historyId},继续等待...`);
return {
status: {
status: 20, // PROCESSING
itemCount: 0,
historyId
} as PollingStatus,
data: { status: 20, item_list: [] }
};
}
const historyData = result[historyId];
const currentStatus = historyData.status;
const currentFailCode = historyData.fail_code;
const currentItemList = historyData.item_list || [];
const finishTime = historyData.task?.finish_time || 0;
// 记录详细信息
if (currentItemList.length > 0) {
const tempVideoUrl = currentItemList[0]?.video?.transcoded_video?.origin?.video_url ||
currentItemList[0]?.video?.play_url ||
currentItemList[0]?.video?.download_url ||
currentItemList[0]?.video?.url;
if (tempVideoUrl) {
logger.info(`检测到视频URL: ${tempVideoUrl}`);
}
}
return {
status: {
status: currentStatus,
failCode: currentFailCode,
itemCount: currentItemList.length,
finishTime,
historyId
} as PollingStatus,
data: historyData
};
}, historyId);
const item_list = finalHistoryData.item_list || [];
// 尝试通过 get_local_item_list 获取高质量视频下载URL
const itemId = item_list?.[0]?.item_id
|| item_list?.[0]?.id
|| item_list?.[0]?.local_item_id
|| item_list?.[0]?.common_attr?.id;
if (itemId) {
try {
const hqVideoUrl = await fetchHighQualityVideoUrl(String(itemId), refreshToken);
if (hqVideoUrl) {
logger.info(`视频生成成功(高质量),URL: ${hqVideoUrl},总耗时: ${pollingResult.elapsedTime}秒`);
return hqVideoUrl;
}
} catch (error) {
logger.warn(`获取高质量视频URL失败,将使用预览URL作为回退: ${error.message}`);
}
} else {
logger.warn(`未能从item_list中提取item_id,将使用预览URL。item_list[0]键: ${item_list?.[0] ? Object.keys(item_list[0]).join(', ') : '无'}`);
}
// 回退:提取预览视频URL
let fallbackVideoUrl = item_list?.[0] ? extractVideoUrl(item_list[0]) : null;
// 如果无法获取视频URL,抛出异常
if (!fallbackVideoUrl) {
logger.error(`未能获取视频URL,item_list: ${JSON.stringify(item_list)}`);
throw new APIException(EX.API_IMAGE_GENERATION_FAILED, "未能获取视频URL,请稍后查看");
}
logger.info(`视频生成成功,URL: ${fallbackVideoUrl},总耗时: ${pollingResult.elapsedTime}秒`);
return fallbackVideoUrl;
}
|