Instructions to use marcosremar2/MuseTalk1.5 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use marcosremar2/MuseTalk1.5 with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("marcosremar2/MuseTalk1.5", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
| import { useState, useRef, useCallback, useEffect } from 'react' | |
| export function useH264Decoder(canvasRef) { | |
| const [isSupported] = useState('VideoDecoder' in window) | |
| const decoderRef = useRef(null) | |
| const frameCountRef = useRef(0) | |
| const initDecoder = useCallback(async (width = 256, height = 256, onFrame) => { | |
| if (!isSupported) return false | |
| if (decoderRef.current) { | |
| decoderRef.current.close() | |
| } | |
| decoderRef.current = new VideoDecoder({ | |
| output: (frame) => { | |
| const canvas = canvasRef.current | |
| if (canvas) { | |
| const ctx = canvas.getContext('2d') | |
| if (canvas.width !== frame.displayWidth) { | |
| canvas.width = frame.displayWidth | |
| canvas.height = frame.displayHeight | |
| } | |
| ctx.drawImage(frame, 0, 0) | |
| } | |
| frame.close() | |
| frameCountRef.current++ | |
| onFrame?.(frameCountRef.current) | |
| }, | |
| error: (e) => console.error('[H264 Decoder] Error:', e) | |
| }) | |
| await decoderRef.current.configure({ | |
| codec: 'avc1.42E01E', | |
| codedWidth: width, | |
| codedHeight: height, | |
| optimizeForLatency: true | |
| }) | |
| return true | |
| }, [isSupported, canvasRef]) | |
| const decodeFrame = useCallback((frameData, isKeyframe) => { | |
| if (!decoderRef.current || decoderRef.current.state === 'closed') return | |
| try { | |
| const chunk = new EncodedVideoChunk({ | |
| type: isKeyframe ? 'key' : 'delta', | |
| timestamp: frameCountRef.current * 40000, | |
| data: frameData | |
| }) | |
| decoderRef.current.decode(chunk) | |
| } catch (e) { | |
| console.error('[H264] Decode error:', e) | |
| } | |
| }, []) | |
| const close = useCallback(() => { | |
| if (decoderRef.current) { | |
| decoderRef.current.close() | |
| decoderRef.current = null | |
| } | |
| frameCountRef.current = 0 | |
| }, []) | |
| useEffect(() => { | |
| return () => close() | |
| }, [close]) | |
| return { isSupported, initDecoder, decodeFrame, close } | |
| } | |