Hugging Face's logo

xiaobinzhuang
/
videofoley

Text-to-Image
Diffusers
StableDiffusionPipeline
Model card Files
xet
 Community
auzxb commited on
Commit
540de95
·
verified ·
1 Parent(s): 37e915b

Upload 13 files

Browse files

diff-foley: projector, latent encoder, audio encoder, video encoder
updates: bigvgan, vae

Files changed (13) hide show
  1. audio_encoder/config.json +6 -0
  2. audio_encoder/diffusion_pytorch_model.bin +3 -0
  3. bigvgan.ckpt +3 -0
  4. latent_encoder/config.json +30 -0
  5. latent_encoder/diffusion_pytorch_model.bin +3 -0
  6. model_index.json +24 -0
  7. projector/config.json +8 -0
  8. projector/diffusion_pytorch_model.bin +3 -0
  9. scheduler/scheduler_config.json +13 -0
  10. vae/config.json +29 -0
  11. vae/diffusion_pytorch_model.bin +3 -0
  12. video_encoder/config.json +6 -0
  13. video_encoder/diffusion_pytorch_model.bin +3 -0
audio_encoder/config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "VisualEncoder",
3
+ "_diffusers_version": "0.18.2",
4
+ "embed_dim": 512
5
+ }
6
+
audio_encoder/diffusion_pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb7790f6ded8d34037b769c8a44191163a92f87989b0a3ba0e709ca6f9080cfb
3
+ size 322984994
bigvgan.ckpt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1da2d4119ca699ec3eaa2e17c0556c362a37cf6257be2db07d5cf680fc9341cc
3
+ size 451301199
latent_encoder/config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "LatentEncoder",
3
+ "_diffusers_version": "0.18.2",
4
+ "attention_resolutions": [
5
+ 4,
6
+ 2,
7
+ 1
8
+ ],
9
+ "channel_mult": [
10
+ 1,
11
+ 2,
12
+ 4,
13
+ 4
14
+ ],
15
+ "context_dim": 768,
16
+ "conv_resample": true,
17
+ "dims": 2,
18
+ "dropout": 0,
19
+ "hidden_channels": 320,
20
+ "in_channels": 4,
21
+ "num_heads": 8,
22
+ "num_res_blocks": 2,
23
+ "out_channels": 4,
24
+ "resblock_updown": false,
25
+ "transformer_depth": 1,
26
+ "use_new_attention_order": false,
27
+ "use_scale_shift_norm": false,
28
+ "use_spatial_transformer": true
29
+ }
30
+
latent_encoder/diffusion_pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:71f0a1796d86e96069ecf1644a1b6081ed62073e38b51e6d36e2f2700081ab68
3
+ size 3438324773
model_index.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "StableDiffusionPipeline",
3
+ "_diffusers_version": "0.6.0",
4
+ "scheduler": [
5
+ "diffusers",
6
+ "PNDMScheduler"
7
+ ],
8
+ "video_encoder": [
9
+ "soundify.cavp.video_model",
10
+ "VideoEncoder"
11
+ ],
12
+ "projector": [
13
+ "soundify.projector.model",
14
+ "Projector"
15
+ ],
16
+ "latent_encoder": [
17
+ "soundify.latent_encoder.model",
18
+ "LatentEncoder"
19
+ ],
20
+ "vae": [
21
+ "diffusers",
22
+ "AutoencoderKL"
23
+ ]
24
+ }
projector/config.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "VisualEncoder",
3
+ "_diffusers_version": "0.18.2",
4
+ "feat_dim": 512,
5
+ "embed_dim": 768,
6
+ "seq_len": 40
7
+ }
8
+
projector/diffusion_pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d018ebcc48f4afea4fd9fad3d58df227d90953fe0b9d9bb6517f78e84c485656
3
+ size 1700275
scheduler/scheduler_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "PNDMScheduler",
3
+ "_diffusers_version": "0.6.0",
4
+ "beta_end": 0.012,
5
+ "beta_schedule": "scaled_linear",
6
+ "beta_start": 0.00085,
7
+ "num_train_timesteps": 1000,
8
+ "set_alpha_to_one": false,
9
+ "skip_prk_steps": true,
10
+ "steps_offset": 1,
11
+ "trained_betas": null,
12
+ "clip_sample": false
13
+ }
vae/config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "AutoencoderKL",
3
+ "_diffusers_version": "0.6.0",
4
+ "act_fn": "silu",
5
+ "block_out_channels": [
6
+ 128,
7
+ 256,
8
+ 512,
9
+ 512
10
+ ],
11
+ "down_block_types": [
12
+ "DownEncoderBlock2D",
13
+ "DownEncoderBlock2D",
14
+ "DownEncoderBlock2D",
15
+ "DownEncoderBlock2D"
16
+ ],
17
+ "in_channels": 3,
18
+ "latent_channels": 4,
19
+ "layers_per_block": 2,
20
+ "norm_num_groups": 32,
21
+ "out_channels": 3,
22
+ "sample_size": 512,
23
+ "up_block_types": [
24
+ "UpDecoderBlock2D",
25
+ "UpDecoderBlock2D",
26
+ "UpDecoderBlock2D",
27
+ "UpDecoderBlock2D"
28
+ ]
29
+ }
vae/diffusion_pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9150709b493d7ee1be671736aeafdd2b53b39766a4acfc14071519d70c52ca0a
3
+ size 334701553
video_encoder/config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "VisualEncoder",
3
+ "_diffusers_version": "0.18.2",
4
+ "embed_dim": 512
5
+ }
6
+
video_encoder/diffusion_pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62d6d2f715927c5d57edac8182ec79be11ccfbb41e2343c007a253ba09749210
3
+ size 131057377