Instructions to use marcosremar2/MuseTalk1.5 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use marcosremar2/MuseTalk1.5 with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("marcosremar2/MuseTalk1.5", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
File size: 1,933 Bytes
32bba92 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | import { useState, useRef, useCallback, useEffect } from 'react'
export function useH264Decoder(canvasRef) {
const [isSupported] = useState('VideoDecoder' in window)
const decoderRef = useRef(null)
const frameCountRef = useRef(0)
const initDecoder = useCallback(async (width = 256, height = 256, onFrame) => {
if (!isSupported) return false
if (decoderRef.current) {
decoderRef.current.close()
}
decoderRef.current = new VideoDecoder({
output: (frame) => {
const canvas = canvasRef.current
if (canvas) {
const ctx = canvas.getContext('2d')
if (canvas.width !== frame.displayWidth) {
canvas.width = frame.displayWidth
canvas.height = frame.displayHeight
}
ctx.drawImage(frame, 0, 0)
}
frame.close()
frameCountRef.current++
onFrame?.(frameCountRef.current)
},
error: (e) => console.error('[H264 Decoder] Error:', e)
})
await decoderRef.current.configure({
codec: 'avc1.42E01E',
codedWidth: width,
codedHeight: height,
optimizeForLatency: true
})
return true
}, [isSupported, canvasRef])
const decodeFrame = useCallback((frameData, isKeyframe) => {
if (!decoderRef.current || decoderRef.current.state === 'closed') return
try {
const chunk = new EncodedVideoChunk({
type: isKeyframe ? 'key' : 'delta',
timestamp: frameCountRef.current * 40000,
data: frameData
})
decoderRef.current.decode(chunk)
} catch (e) {
console.error('[H264] Decode error:', e)
}
}, [])
const close = useCallback(() => {
if (decoderRef.current) {
decoderRef.current.close()
decoderRef.current = null
}
frameCountRef.current = 0
}, [])
useEffect(() => {
return () => close()
}, [close])
return { isSupported, initDecoder, decodeFrame, close }
}
|