server-plugins/single-static-video.js

import path from 'path';
import fs from 'fs';
import { Plugin } from './plugin.js';
import { FFMpegUtils } from 'common-utils';
import { CaptionPlugin } from './generate-captions.js';

/**
 * Must come before the Captions plugin in the plugin list, since it modifies the transcript and bubbles before captions are generated.
 * The single static video plugin aims to join all the audio segments of a transcript into a single audio file. Then use the static video in the first section of the transcript as a placeholder for the entire transcript. This is useful for cases where the user wants to use a single static video for the entire transcript. Basically transforming all the sections into a single section with a single static video and a single audio file. The static video is used as a placeholder for the entire transcript, and the audio is used to play the entire transcript. The static video is not modified in any way, it is simply used as a placeholder for the entire transcript. The audio is generated by concatenating all the audio segments of the transcript into a single audio file. The output of this plugin is a modified transcript with a single section containing the static video and the concatenated audio file. The bubbles across all sections are also moved to the single combined section by reading each section duration, bubble duration and addind appropriate offsets to the bubble start and end times.  The output of this plugin is a modified transcript with a single section containing the static video and the concatenated audio file, and the bubbles across all sections are also moved to the single combined section by reading each section duration, bubble duration and adding appropriate offsets to the bubble start and end times. The bubbles are also sorted by their start time. Transitions are removed during this process.
 */
export class SingleStaticVideoPlugin extends Plugin {
  constructor(name, options) {
    super(name, options);
  }


  async applyPrerender(originalManuscript, jobId) {
    const transcript = originalManuscript.transcript || [];
    const originalManuscriptMeta = originalManuscript.meta || [];
    const targetWidth = +this.options.width || +this.options.targetWidth || originalManuscriptMeta?.resolution?.width || 1080;
    const targetHeight = +this.options.height || +this.options.targetHeight || originalManuscriptMeta?.resolution?.height || 1920;
    const targetAspect = targetWidth / targetHeight;
    let firstPlaceholderMedia = undefined
    let firstPlaceholderMeta = undefined

    let combinedAudioPath = path.join('public', `combined-audio-${jobId}.mp3`)
    let combinedAudioCaptionsFilePath = path.join('public', `combined-audio-captions-${jobId}.json`)
    let combinedBubbles = []
    let combinedMediaPath = undefined
    let combinedMediaMeta = undefined
    let allAudios = []

    for (let item of transcript) {

      let sectionidx = transcript.indexOf(item)
      if (!item.mediaAbsPaths || !item.mediaAbsPaths.length) continue;
      allAudios.push(item.audioFullPath)


      for (let mediaObj of item.mediaAbsPaths) {
        if (!combinedMediaPath) {
          // We only need to process the first section with media, since we are using the first media from the first section for single static video for the entire transcript
          let mediaIdx = item.mediaAbsPaths.indexOf(mediaObj)
          try {
            let mediaPath = mediaObj.path;
            if (!mediaPath || !fs.existsSync(mediaPath)) {
              const flattenedPath = this.mediaPathFlatten(mediaPath);
              if (fs.existsSync(flattenedPath)) {
                mediaObj.path = flattenedPath;
                mediaPath = flattenedPath;
                this.log(`Using flattened media path: ${flattenedPath}`);
              } else {
                this.log(`Media path does not exist: ${mediaPath}. Trying at flattened path: ${flattenedPath} failed.`);
                continue;
              }
            }
            // The first media path is used as the placeholder for the entire transcript. This is useful for cases where the user wants to use a single static video for the entire transcript
            if (!firstPlaceholderMedia) {
              firstPlaceholderMedia = mediaPath
              firstPlaceholderMeta = await FFMpegUtils.getMediaMetadata(mediaPath);
              if (!firstPlaceholderMeta || !firstPlaceholderMeta.video || !firstPlaceholderMeta.video.width || !firstPlaceholderMeta.video.height) {
                this.log(`No video stream found for ${firstPlaceholderMedia}`);
                continue;
              }
            }

            combinedMediaMeta = firstPlaceholderMeta
            const ext = path.extname(mediaPath);
            const base = path.basename(mediaPath, ext);
            combinedMediaPath = path.join(path.dirname(mediaPath), `combined-static-${sectionidx}-${mediaIdx}${ext}`);

            mediaObj.path = combinedMediaPath;
          } catch (err) {
            this.log(`Error cropping media ${mediaObj?.path}: ${err}`);
          }
        }
      }
    }

    combinedBubbles = await this.combineAllTranscriptBubbles(transcript, {
      contagious: this.options.contagious || false,
      contagiousDelay: this.options.contagiousDelay ?? 0
    })
    await Promise.all([
      this.combineAllAudioCaptions(transcript, combinedAudioCaptionsFilePath),
      this.combineAllAudios(transcript, combinedAudioPath)
    ])

    // Calculate total duration across all sections
    let totalDuration = 0;
    for (let item of transcript) {
      totalDuration += item.durationInSeconds || 0;
    }

    await this.combineFirstPlaceholderMedia(firstPlaceholderMedia, combinedAudioPath, combinedMediaPath, totalDuration, firstPlaceholderMeta);
    // Build the single combined section from the first section as a template
    const firstSection = transcript[0] || {};
    const combinedSection = {
      ...firstSection,
      text: transcript.map(s => s.text || '').join(' '),
      index: 0,
      mediaAbsPaths: combinedMediaPath ? [{
        path: combinedMediaPath,
        type: firstSection.mediaAbsPaths?.[0]?.type || 'video',
        dimensions: combinedMediaMeta?.video ? {
          width: parseInt(combinedMediaMeta.video.width),
          height: parseInt(combinedMediaMeta.video.height)
        } : undefined,
        durationSec: totalDuration
      }] : [],
      bubbles: combinedBubbles,
      audioFullPath: combinedAudioPath,
      audioCaptionFile: combinedAudioCaptionsFilePath,
      durationInSeconds: totalDuration,
      duration: Math.ceil(totalDuration * (originalManuscriptMeta?.fps || 30)),
      offset: 0,
      // Remove transitions since everything is merged into one section
      transition_type: 'none',
      transition_file: undefined,
      transition_duration_sec: 0
    };

    // Replace the transcript with the single combined section
    originalManuscript.transcript = [combinedSection];

    this.log(`Combined ${transcript.length} sections into 1 static-video section. Total duration: ${totalDuration.toFixed(2)}s, Bubbles: ${combinedBubbles.length}`);
  }

  async combineFirstPlaceholderMedia(inputPath, audioPath, outputPath, totalDuration, inputMeta = null) {
    if (!inputPath || !fs.existsSync(inputPath)) {
      this.log(`Placeholder media not found: ${inputPath}`);
      return;
    }

    this.log(`Clipping/Looping placeholder media ${inputPath} to ${totalDuration}s -> ${outputPath}`);
    try {
      const ext = path.extname(inputPath).toLowerCase();
      const isImage = ['.png', '.jpg', '.jpeg', '.webp'].includes(ext);

      let cmd = '';
      if (isImage) {
        cmd = `ffmpeg -loop 1 -i "${inputPath}" -i "${audioPath}" -t ${totalDuration} -c:v libx264 -preset veryfast -crf 23 -pix_fmt yuv420p -c:a aac -map 0:v -map 1:a "${outputPath}" -y`;
      } else {
        let inputDuration = 0;
        if (inputMeta && inputMeta.format && inputMeta.format.duration) {
          inputDuration = parseFloat(inputMeta.format.duration);
        } else {
          try {
            const meta = await FFMpegUtils.getMediaMetadata(inputPath);
            inputDuration = parseFloat(meta?.format?.duration || 0);
          } catch (e) {
            this.log(`Could not fetch metadata for duration: ${e}`);
          }
        }

        if (inputDuration > totalDuration) {
          const startTime = Math.max(0, inputDuration - totalDuration);
          cmd = `ffmpeg -ss ${startTime} -i "${inputPath}" -i "${audioPath}" -t ${totalDuration} -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 "${outputPath}" -y`;
        } else {
          cmd = `ffmpeg -stream_loop -1 -i "${inputPath}" -i "${audioPath}" -t ${totalDuration} -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 "${outputPath}" -y`;
        }
      }

      await FFMpegUtils.execute(cmd);
    } catch (err) {
      this.log(`Error in combineFirstPlaceholderMedia: ${err}`);
      // fallback to copy if ffmpeg fails
      fs.copyFileSync(inputPath, outputPath);
    }
  }

  // Takes all transcripts and returns a single array with bubbles from different sections put into a single array and sorted by start time. The bubbles are also adjusted to have the correct start and end times based on the section duration and bubble duration.  Return array of bubbles.
  // Options:
  //   contagious {boolean} - if true, each bubble[N+1].fromSec is forced to bubble[N].toSec + contagiousDelay after sorting
  //   contagiousDelay {number} - gap in seconds between consecutive bubbles when contagious=true (default: 0)
  async combineAllTranscriptBubbles(transcripts, { contagious = false, contagiousDelay = 0 } = {}) {
    let combinedBubbles = [];
    let cumulativeOffset = 0;

    for (let section of transcripts) {
      const sectionDuration = section.durationInSeconds || 0;
      const bubbles = section.bubbles || [];

      for (let bubble of bubbles) {
        // Deep clone the bubble so we don't mutate the original
        const adjustedBubble = JSON.parse(JSON.stringify(bubble));

        // Adjust timing with cumulative offset from previous sections
        if (adjustedBubble.fromSec !== undefined && adjustedBubble.fromSec !== null) {
          adjustedBubble.fromSec = adjustedBubble.fromSec + cumulativeOffset;
        }
        if (adjustedBubble.toSec !== undefined && adjustedBubble.toSec !== null) {
          adjustedBubble.toSec = adjustedBubble.toSec + cumulativeOffset;
        }

        combinedBubbles.push(adjustedBubble);
      }

      cumulativeOffset += sectionDuration;
    }

    // Sort bubbles by their start time
    combinedBubbles.sort((a, b) => (a.fromSec || 0) - (b.fromSec || 0));

    // Apply contagious chaining: bubble[N+1].fromSec = bubble[N].toSec + contagiousDelay
    if (contagious && combinedBubbles.length > 1) {
      this.log(`Applying contagious chaining with delay=${contagiousDelay}s across ${combinedBubbles.length} bubbles.`);
      for (let i = 1; i < combinedBubbles.length; i++) {
        const prev = combinedBubbles[i - 1];
        if (prev.toSec !== undefined && prev.toSec !== null) {
          const newFromSec = prev.toSec + contagiousDelay;
          if (combinedBubbles[i].fromSec !== undefined && combinedBubbles[i].toSec !== undefined) {
            // Preserve original duration, shift the window forward
            const originalDur = combinedBubbles[i].toSec - combinedBubbles[i].fromSec;
            combinedBubbles[i].fromSec = newFromSec;
            combinedBubbles[i].toSec = Math.round((newFromSec + originalDur) * 1000) / 1000;
          } else {
            combinedBubbles[i].fromSec = newFromSec;
          }
          // Mirror into mediaTextPrompts if present
          if (combinedBubbles[i].mediaTextPrompts?.length) {
            combinedBubbles[i].mediaTextPrompts[0].fromSec = combinedBubbles[i].fromSec;
            combinedBubbles[i].mediaTextPrompts[0].toSec = combinedBubbles[i].toSec;
            if (combinedBubbles[i].durationSec !== undefined) {
              combinedBubbles[i].durationSec = Math.round((combinedBubbles[i].toSec - combinedBubbles[i].fromSec) * 1000) / 1000;
              combinedBubbles[i].mediaTextPrompts[0].durationSec = combinedBubbles[i].durationSec;
            }
          }
        }
      }
    }

    this.log(`Combined ${combinedBubbles.length} bubbles from ${transcripts.length} sections`);
    return combinedBubbles;
  }

  // Combines all audio files from all transcripts into a single audio file.  The combined audio file is saved to the specified output path.
  async combineAllAudios(transcripts, outAudioPath) {
    // flatten path first for each audio before processing this.mediaPathFlatten(..);
    const audioPaths = [];

    for (let section of transcripts) {
      let audioPath = section.audioFullPath;
      if (!audioPath) continue;

      // Try the original path first, then fall back to flattened
      if (!fs.existsSync(audioPath)) {
        const flattenedPath = this.mediaPathFlatten(audioPath);
        if (fs.existsSync(flattenedPath)) {
          audioPath = flattenedPath;
          this.log(`Using flattened audio path: ${flattenedPath}`);
        } else {
          this.log(`Audio path does not exist: ${audioPath}. Flattened path ${flattenedPath} also not found. Skipping.`);
          continue;
        }
      }

      audioPaths.push(audioPath);
    }

    if (audioPaths.length === 0) {
      this.log('No audio files found to combine.');
      return;
    }

    if (audioPaths.length === 1) {
      // Only one audio file, just copy it
      fs.copyFileSync(audioPaths[0], outAudioPath);
      this.log(`Single audio file copied to ${outAudioPath}`);
      return;
    }

    this.log(`Joining ${audioPaths.length} audio files into ${outAudioPath}`);
    await FFMpegUtils.joinAudios(audioPaths, outAudioPath);
    this.log(`Combined audio saved to ${outAudioPath}`);
  }

  // Combines all audio captions for all transcripts into a single audio caption file taking care of offset. The combined audio caption file is saved to the specified output path.
  async combineAllAudioCaptions(transcripts, outAudioCaptionPath) {
    // flatten path first for each audio caption file before processing this.mediaPathFlatten(..);
    let combinedTranscriptText = '';
    let combinedWords = [];
    let cumulativeOffset = 0;

    for (let section of transcripts) {
      let captionFilePath = section.audioCaptionFile;
      if (!captionFilePath) {
        // No caption file for this section, just advance the offset
        cumulativeOffset += section.durationInSeconds || 0;
        continue;
      }

      // If the caption file is an .ass file, check for the original JSON source
      if (path.extname(captionFilePath) === '.ass') {
        // Try the _audioCaptionFile (original JSON before ASS conversion) if available
        if (section._audioCaptionFile) {
          captionFilePath = section._audioCaptionFile;
        } else {
          // Try converting .ass path back to .json
          captionFilePath = captionFilePath.replace('.ass', '.json');
        }
      }

      // Resolve the caption file path
      let resolvedPath = captionFilePath;
      if (!fs.existsSync(resolvedPath)) {
        // Try with absolute path from cwd
        resolvedPath = path.join(process.cwd(), captionFilePath);
      }
      if (!fs.existsSync(resolvedPath)) {
        // Try flattened path
        const flattenedPath = this.mediaPathFlatten(captionFilePath);
        if (fs.existsSync(flattenedPath)) {
          resolvedPath = flattenedPath;
          this.log(`Using flattened caption path: ${flattenedPath}`);
        } else {
          const flattenedAbsPath = path.join(process.cwd(), flattenedPath);
          if (fs.existsSync(flattenedAbsPath)) {
            resolvedPath = flattenedAbsPath;
            this.log(`Using flattened absolute caption path: ${flattenedAbsPath}`);
          } else {
            this.log(`Caption file not found: ${captionFilePath}. Skipping.`);
            cumulativeOffset += section.durationInSeconds || 0;
            continue;
          }
        }
      }

      try {
        const captionData = JSON.parse(fs.readFileSync(resolvedPath, 'utf-8'));
        const sectionTranscript = captionData.transcript || '';
        const sectionWords = captionData.words || [];

        // Append transcript text with a space separator
        if (combinedTranscriptText.length > 0 && sectionTranscript.length > 0) {
          combinedTranscriptText += ' ';
        }
        combinedTranscriptText += sectionTranscript;

        // Adjust word timings with cumulative offset and add to combined list
        for (let word of sectionWords) {
          const adjustedWord = { ...word };
          if (adjustedWord.start !== undefined && adjustedWord.start !== null) {
            adjustedWord.start = adjustedWord.start + cumulativeOffset;
          }
          if (adjustedWord.end !== undefined && adjustedWord.end !== null) {
            adjustedWord.end = adjustedWord.end + cumulativeOffset;
          }
          combinedWords.push(adjustedWord);
        }
      } catch (err) {
        this.log(`Error reading caption file ${resolvedPath}: ${err}`);
      }

      cumulativeOffset += section.durationInSeconds || 0;
    }

    // Write the combined caption file
    const combinedCaptions = {
      transcript: combinedTranscriptText,
      words: combinedWords
    };

    fs.writeFileSync(outAudioCaptionPath, JSON.stringify(combinedCaptions, null, 2));
    this.log(`Combined ${combinedWords.length} caption words from ${transcripts.length} sections into ${outAudioCaptionPath}`);
  }


}