Ava2lon commited on
Commit
256ece2
·
verified ·
1 Parent(s): 1041911

Upload 7 files

Browse files
docs/images/icon.png ADDED
docs/index.html ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <!-- Google tag (gtag.js) -->
5
+ <script async src="https://www.googletagmanager.com/gtag/js?id=G-0JKBJ3WRJZ"></script>
6
+ <script>
7
+ window.dataLayer = window.dataLayer || [];
8
+ function gtag(){dataLayer.push(arguments);}
9
+ gtag('js', new Date());
10
+ gtag('config', 'G-0JKBJ3WRJZ');
11
+ </script>
12
+
13
+ <link rel="preconnect" href="https://fonts.googleapis.com">
14
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
15
+ <link href="https://fonts.googleapis.com/css2?family=Source+Sans+3&display=swap" rel="stylesheet">
16
+ <meta charset="UTF-8">
17
+ <title>MMAudio</title>
18
+
19
+ <link rel="icon" type="image/png" href="images/icon.png">
20
+
21
+ <meta name="viewport" content="width=device-width, initial-scale=1">
22
+ <!-- CSS only -->
23
+ <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.1/dist/css/bootstrap.min.css" rel="stylesheet"
24
+ integrity="sha384-+0n0xVW2eSR5OomGNYDnhzAbDsOXxcvSN1TPprVMTNDbiYZCxYbOOl7+AMvyTG2x" crossorigin="anonymous">
25
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
26
+
27
+ <link rel="stylesheet" href="style.css">
28
+ </head>
29
+ <body>
30
+
31
+ <body>
32
+ <br><br><br><br>
33
+ <div class="container">
34
+ <div class="row text-center" style="font-size:38px">
35
+ <div class="col strong">
36
+ Taming Multimodal Joint Training for High-Quality <br>Video-to-Audio Synthesis
37
+ </div>
38
+ </div>
39
+
40
+ <br>
41
+ <div class="row text-center" style="font-size:28px">
42
+ <div class="col">
43
+ arXiv 2024
44
+ </div>
45
+ </div>
46
+ <br>
47
+
48
+ <div class="h-100 row text-center heavy justify-content-md-center" style="font-size:22px;">
49
+ <div class="col-sm-auto px-lg-2">
50
+ <a href="https://hkchengrex.github.io/">Ho Kei Cheng<sup>1</sup></a>
51
+ </div>
52
+ <div class="col-sm-auto px-lg-2">
53
+ <nobr><a href="https://scholar.google.co.jp/citations?user=RRIO1CcAAAAJ">Masato Ishii<sup>2</sup></a></nobr>
54
+ </div>
55
+ <div class="col-sm-auto px-lg-2">
56
+ <nobr><a href="https://scholar.google.com/citations?user=sXAjHFIAAAAJ">Akio Hayakawa<sup>2</sup></a></nobr>
57
+ </div>
58
+ <div class="col-sm-auto px-lg-2">
59
+ <nobr><a href="https://scholar.google.com/citations?user=XCRO260AAAAJ">Takashi Shibuya<sup>2</sup></a></nobr>
60
+ </div>
61
+ <div class="col-sm-auto px-lg-2">
62
+ <nobr><a href="https://www.alexander-schwing.de/">Alexander Schwing<sup>1</sup></a></nobr>
63
+ </div>
64
+ <div class="col-sm-auto px-lg-2" >
65
+ <nobr><a href="https://www.yukimitsufuji.com/">Yuki Mitsufuji<sup>2,3</sup></a></nobr>
66
+ </div>
67
+ </div>
68
+
69
+ <div class="h-100 row text-center heavy justify-content-md-center" style="font-size:22px;">
70
+ <div class="col-sm-auto px-lg-2">
71
+ <sup>1</sup>University of Illinois Urbana-Champaign
72
+ </div>
73
+ <div class="col-sm-auto px-lg-2">
74
+ <sup>2</sup>Sony AI
75
+ </div>
76
+ <div class="col-sm-auto px-lg-2">
77
+ <sup>3</sup>Sony Group Corporation
78
+ </div>
79
+ </div>
80
+
81
+ <br>
82
+
83
+ <br>
84
+
85
+ <div class="h-100 row text-center justify-content-md-center" style="font-size:20px;">
86
+ <!-- <div class="col-sm-2">
87
+ <a href="https://arxiv.org/abs/2310.12982">[arXiv]</a>
88
+ </div> -->
89
+ <div class="col-sm-3">
90
+ <a href="">[Paper (being prepared)]</a>
91
+ </div>
92
+ <div class="col-sm-3">
93
+ <a href="https://github.com/hkchengrex/MMAudio">[Code]</a>
94
+ </div>
95
+ <!-- <div class="col-sm-2">
96
+ <a
97
+ href="https://colab.research.google.com/drive/1yo43XTbjxuWA7XgCUO9qxAi7wBI6HzvP?usp=sharing">[Colab]</a>
98
+ </div> -->
99
+ </div>
100
+
101
+ <br>
102
+
103
+ <hr>
104
+
105
+ <div class="row" style="font-size:32px">
106
+ <div class="col strong">
107
+ TL;DR
108
+ </div>
109
+ </div>
110
+ <br>
111
+ <div class="row">
112
+ <div class="col">
113
+ <p class="light" style="text-align: left;">
114
+ MMAudio generates synchronized audio given video and/or text inputs.
115
+ </p>
116
+ </div>
117
+ </div>
118
+
119
+ <br>
120
+ <hr>
121
+ <br>
122
+
123
+ <div class="row" style="font-size:32px">
124
+ <div class="col strong">
125
+ Demo
126
+ </div>
127
+ </div>
128
+ <br>
129
+ <div class="row" style="font-size:48px">
130
+ <div class="col strong text-center">
131
+ <a href="video_main.html" style="text-decoration: underline;">&lt;More results&gt;</a>
132
+ </div>
133
+ </div>
134
+ <br>
135
+ <div class="video-container" style="text-align: center;">
136
+ <iframe src="https://youtube.com/embed/YElewUT2M4M"></iframe>
137
+ </div>
138
+
139
+ <br>
140
+
141
+ <br><br>
142
+ <br><br>
143
+
144
+ </div>
145
+
146
+ </body>
147
+ </html>
docs/style.css ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ body {
2
+ font-family: 'Source Sans 3', sans-serif;
3
+ font-size: 18px;
4
+ margin-left: auto;
5
+ margin-right: auto;
6
+ font-weight: 400;
7
+ height: 100%;
8
+ max-width: 1000px;
9
+ }
10
+
11
+ table {
12
+ width: 100%;
13
+ border-collapse: collapse;
14
+ }
15
+ th, td {
16
+ border: 1px solid #ddd;
17
+ padding: 8px;
18
+ text-align: center;
19
+ }
20
+ th {
21
+ background-color: #f2f2f2;
22
+ }
23
+ video {
24
+ width: 100%;
25
+ height: auto;
26
+ }
27
+ p {
28
+ font-size: 28px;
29
+ }
30
+ h2 {
31
+ font-size: 36px;
32
+ }
33
+
34
+ .strong {
35
+ font-weight: 700;
36
+ }
37
+
38
+ .light {
39
+ font-weight: 100;
40
+ }
41
+
42
+ .heavy {
43
+ font-weight: 900;
44
+ }
45
+
46
+ .column {
47
+ float: left;
48
+ }
49
+
50
+ a:link,
51
+ a:visited {
52
+ color: #05538f;
53
+ text-decoration: none;
54
+ }
55
+
56
+ a:hover {
57
+ color: #63cbdd;
58
+ }
59
+
60
+ hr {
61
+ border: 0;
62
+ height: 1px;
63
+ background-image: linear-gradient(to right, rgba(0, 0, 0, 0), rgba(0, 0, 0, 0.75), rgba(0, 0, 0, 0));
64
+ }
65
+
66
+ .video-container {
67
+ position: relative;
68
+ padding-bottom: 56.25%; /* 16:9 */
69
+ height: 0;
70
+ }
71
+
72
+ .video-container iframe {
73
+ position: absolute;
74
+ top: 0;
75
+ left: 0;
76
+ width: 100%;
77
+ height: 100%;
78
+ }
docs/style_videos.css ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ body {
2
+ font-family: 'Source Sans 3', sans-serif;
3
+ font-size: 1.5vh;
4
+ font-weight: 400;
5
+ }
6
+
7
+ table {
8
+ width: 100%;
9
+ border-collapse: collapse;
10
+ }
11
+ th, td {
12
+ border: 1px solid #ddd;
13
+ padding: 8px;
14
+ text-align: center;
15
+ }
16
+ th {
17
+ background-color: #f2f2f2;
18
+ }
19
+ video {
20
+ width: 100%;
21
+ height: auto;
22
+ }
23
+ p {
24
+ font-size: 1.5vh;
25
+ font-weight: bold;
26
+ }
27
+ h2 {
28
+ font-size: 2vh;
29
+ font-weight: bold;
30
+ }
31
+
32
+ .video-container {
33
+ position: relative;
34
+ padding-bottom: 56.25%; /* 16:9 */
35
+ height: 0;
36
+ }
37
+
38
+ .video-container iframe {
39
+ position: absolute;
40
+ top: 0;
41
+ left: 0;
42
+ width: 100%;
43
+ height: 100%;
44
+ }
45
+
46
+ .video-header {
47
+ background-color: #f2f2f2;
48
+ text-align: center;
49
+ font-size: 1.5vh;
50
+ font-weight: bold;
51
+ padding: 8px;
52
+ }
docs/video_gen.html ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <!-- Google tag (gtag.js) -->
5
+ <script async src="https://www.googletagmanager.com/gtag/js?id=G-0JKBJ3WRJZ"></script>
6
+ <script>
7
+ window.dataLayer = window.dataLayer || [];
8
+ function gtag(){dataLayer.push(arguments);}
9
+ gtag('js', new Date());
10
+ gtag('config', 'G-0JKBJ3WRJZ');
11
+ </script>
12
+
13
+ <link href='https://fonts.googleapis.com/css?family=Source+Sans+Pro' rel='stylesheet' type='text/css'>
14
+ <meta charset="UTF-8">
15
+ <title>MMAudio</title>
16
+
17
+ <link rel="icon" type="image/png" href="images/icon.png">
18
+
19
+ <meta name="viewport" content="width=device-width, initial-scale=1">
20
+ <!-- CSS only -->
21
+ <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.1/dist/css/bootstrap.min.css" rel="stylesheet"
22
+ integrity="sha384-+0n0xVW2eSR5OomGNYDnhzAbDsOXxcvSN1TPprVMTNDbiYZCxYbOOl7+AMvyTG2x" crossorigin="anonymous">
23
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.7.1/jquery.min.js"></script>
24
+
25
+ <link rel="stylesheet" href="style_videos.css">
26
+ </head>
27
+ <body>
28
+
29
+ <div id="moviegen_all">
30
+ <h2 id="moviegen" style="text-align: center;">Comparisons with Movie Gen Audio on Videos Generated by MovieGen</h2>
31
+ <p id="moviegen1" style="overflow: hidden;">
32
+ Example 1: Ice cracking with sharp snapping sound, and metal tool scraping against the ice surface.
33
+ <span style="float: right;"><a href="#index">Back to index</a></span>
34
+ </p>
35
+
36
+ <div class="row g-1">
37
+ <div class="col-sm-6">
38
+ <div class="video-header">Movie Gen Audio</div>
39
+ <div class="video-container">
40
+ <iframe src="https://youtube.com/embed/d7Lb0ihtGcE"></iframe>
41
+ </div>
42
+ </div>
43
+ <div class="col-sm-6">
44
+ <div class="video-header">Ours</div>
45
+ <div class="video-container">
46
+ <iframe src="https://youtube.com/embed/F4JoJ2r2m8U"></iframe>
47
+ </div>
48
+ </div>
49
+ </div>
50
+ <br>
51
+
52
+ <!-- <p id="moviegen2">Example 2: Rhythmic splashing and lapping of water. <span style="float:right;"><a href="#index">Back to index</a></span> </p>
53
+
54
+ <table>
55
+ <thead>
56
+ <tr>
57
+ <th>Movie Gen Audio</th>
58
+ <th>Ours</th>
59
+ </tr>
60
+ </thead>
61
+ <tbody>
62
+ <tr>
63
+ <td width="50%">
64
+ <div class="video-container">
65
+ <iframe src="https://youtube.com/embed/5gQNPK99CIk"></iframe>
66
+ </div>
67
+ </td>
68
+ <td width="50%">
69
+ <div class="video-container">
70
+ <iframe src="https://youtube.com/embed/AbwnTzG-BpA"></iframe>
71
+ </div>
72
+ </td>
73
+ </tr>
74
+ </tbody>
75
+ </table> -->
76
+
77
+ <p id="moviegen2" style="overflow: hidden;">
78
+ Example 2: Rhythmic splashing and lapping of water.
79
+ <span style="float:right;"><a href="#index">Back to index</a></span>
80
+ </p>
81
+ <div class="row g-1">
82
+ <div class="col-sm-6">
83
+ <div class="video-header">Movie Gen Audio</div>
84
+ <div class="video-container">
85
+ <iframe src="https://youtube.com/embed/5gQNPK99CIk"></iframe>
86
+ </div>
87
+ </div>
88
+ <div class="col-sm-6">
89
+ <div class="video-header">Ours</div>
90
+ <div class="video-container">
91
+ <iframe src="https://youtube.com/embed/AbwnTzG-BpA"></iframe>
92
+ </div>
93
+ </div>
94
+ </div>
95
+ <br>
96
+
97
+ <p id="moviegen3" style="overflow: hidden;">
98
+ Example 3: Shovel scrapes against dry earth.
99
+ <span style="float:right;"><a href="#index">Back to index</a></span>
100
+ </p>
101
+ <div class="row g-1">
102
+ <div class="col-sm-6">
103
+ <div class="video-header">Movie Gen Audio</div>
104
+ <div class="video-container">
105
+ <iframe src="https://youtube.com/embed/PUKGyEve7XQ"></iframe>
106
+ </div>
107
+ </div>
108
+ <div class="col-sm-6">
109
+ <div class="video-header">Ours</div>
110
+ <div class="video-container">
111
+ <iframe src="https://youtube.com/embed/CNn7i8VNkdc"></iframe>
112
+ </div>
113
+ </div>
114
+ </div>
115
+ <br>
116
+
117
+
118
+ <p id="moviegen4" style="overflow: hidden;">
119
+ (Failure case) Example 4: Creamy sound of mashed potatoes being scooped.
120
+ <span style="float:right;"><a href="#index">Back to index</a></span>
121
+ </p>
122
+ <div class="row g-1">
123
+ <div class="col-sm-6">
124
+ <div class="video-header">Movie Gen Audio</div>
125
+ <div class="video-container">
126
+ <iframe src="https://youtube.com/embed/PJv1zxR9JjQ"></iframe>
127
+ </div>
128
+ </div>
129
+ <div class="col-sm-6">
130
+ <div class="video-header">Ours</div>
131
+ <div class="video-container">
132
+ <iframe src="https://youtube.com/embed/c3-LJ1lNsPQ"></iframe>
133
+ </div>
134
+ </div>
135
+ </div>
136
+ <br>
137
+
138
+ </div>
139
+
140
+ <div id="hunyuan_sora_all">
141
+
142
+ <h2 id="hunyuan" style="text-align: center;">Results on Videos Generated by Hunyuan</h2>
143
+ <p style="overflow: hidden;">
144
+ <span style="float:right;"><a href="#index">Back to index</a></span>
145
+ </p>
146
+ <div class="row g-1">
147
+ <div class="col-sm-6">
148
+ <div class="video-header">Typing</div>
149
+ <div class="video-container">
150
+ <iframe src="https://youtube.com/embed/8ln_9hhH_nk"></iframe>
151
+ </div>
152
+ </div>
153
+ <div class="col-sm-6">
154
+ <div class="video-header">Water is rushing down a stream and pouring</div>
155
+ <div class="video-container">
156
+ <iframe src="https://youtube.com/embed/5df1FZFQj30"></iframe>
157
+ </div>
158
+ </div>
159
+ </div>
160
+ <div class="row g-1">
161
+ <div class="col-sm-6">
162
+ <div class="video-header">Waves on beach</div>
163
+ <div class="video-container">
164
+ <iframe src="https://youtube.com/embed/7wQ9D5WgpFc"></iframe>
165
+ </div>
166
+ </div>
167
+ <div class="col-sm-6">
168
+ <div class="video-header">Water droplet</div>
169
+ <div class="video-container">
170
+ <iframe src="https://youtube.com/embed/q7M2nsalGjM"></iframe>
171
+ </div>
172
+ </div>
173
+ </div>
174
+ <br>
175
+
176
+ <h2 id="sora" style="text-align: center;">Results on Videos Generated by Sora</h2>
177
+ <p style="overflow: hidden;">
178
+ <span style="float:right;"><a href="#index">Back to index</a></span>
179
+ </p>
180
+ <div class="row g-1">
181
+ <div class="col-sm-6">
182
+ <div class="video-header">Ships riding waves</div>
183
+ <div class="video-container">
184
+ <iframe src="https://youtube.com/embed/JbgQzHHytk8"></iframe>
185
+ </div>
186
+ </div>
187
+ <div class="col-sm-6">
188
+ <div class="video-header">Train (no text prompt given)</div>
189
+ <div class="video-container">
190
+ <iframe src="https://youtube.com/embed/xOW7zrjpWC8"></iframe>
191
+ </div>
192
+ </div>
193
+ </div>
194
+ <div class="row g-1">
195
+ <div class="col-sm-6">
196
+ <div class="video-header">Seashore (no text prompt given)</div>
197
+ <div class="video-container">
198
+ <iframe src="https://youtube.com/embed/fIuw5Y8ZZ9E"></iframe>
199
+ </div>
200
+ </div>
201
+ <div class="col-sm-6">
202
+ <div class="video-header">Surfing (failure: unprompted music)</div>
203
+ <div class="video-container">
204
+ <iframe src="https://youtube.com/embed/UcSTk-v0M_s"></iframe>
205
+ </div>
206
+ </div>
207
+ </div>
208
+ <br>
209
+
210
+ <div id="mochi_ltx_all">
211
+ <h2 id="mochi" style="text-align: center;">Results on Videos Generated by Mochi 1</h2>
212
+ <p style="overflow: hidden;">
213
+ <span style="float:right;"><a href="#index">Back to index</a></span>
214
+ </p>
215
+ <div class="row g-1">
216
+ <div class="col-sm-6">
217
+ <div class="video-header">Magical fire and lightning (no text prompt given)</div>
218
+ <div class="video-container">
219
+ <iframe src="https://youtube.com/embed/tTlRZaSMNwY"></iframe>
220
+ </div>
221
+ </div>
222
+ <div class="col-sm-6">
223
+ <div class="video-header">Storm (no text prompt given)</div>
224
+ <div class="video-container">
225
+ <iframe src="https://youtube.com/embed/4hrZTMJUy3w"></iframe>
226
+ </div>
227
+ </div>
228
+ </div>
229
+ <br>
230
+
231
+ <h2 id="ltx" style="text-align: center;">Results on Videos Generated by LTX-Video</h2>
232
+ <p style="overflow: hidden;">
233
+ <span style="float:right;"><a href="#index">Back to index</a></span>
234
+ </p>
235
+ <div class="row g-1">
236
+ <div class="col-sm-6">
237
+ <div class="video-header">Firewood burning and cracking</div>
238
+ <div class="video-container">
239
+ <iframe src="https://youtube.com/embed/P7_DDpgev0g"></iframe>
240
+ </div>
241
+ </div>
242
+ <div class="col-sm-6">
243
+ <div class="video-header">Waterfall, water splashing</div>
244
+ <div class="video-container">
245
+ <iframe src="https://youtube.com/embed/4MvjceYnIO0"></iframe>
246
+ </div>
247
+ </div>
248
+ </div>
249
+ <br>
250
+
251
+ </div>
252
+
253
+ </body>
254
+ </html>
docs/video_main.html ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <!-- Google tag (gtag.js) -->
5
+ <script async src="https://www.googletagmanager.com/gtag/js?id=G-0JKBJ3WRJZ"></script>
6
+ <script>
7
+ window.dataLayer = window.dataLayer || [];
8
+ function gtag(){dataLayer.push(arguments);}
9
+ gtag('js', new Date());
10
+ gtag('config', 'G-0JKBJ3WRJZ');
11
+ </script>
12
+
13
+ <link href='https://fonts.googleapis.com/css?family=Source+Sans+Pro' rel='stylesheet' type='text/css'>
14
+ <meta charset="UTF-8">
15
+ <title>MMAudio</title>
16
+
17
+ <link rel="icon" type="image/png" href="images/icon.png">
18
+
19
+ <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
20
+ <!-- CSS only -->
21
+ <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.1/dist/css/bootstrap.min.css" rel="stylesheet"
22
+ integrity="sha384-+0n0xVW2eSR5OomGNYDnhzAbDsOXxcvSN1TPprVMTNDbiYZCxYbOOl7+AMvyTG2x" crossorigin="anonymous">
23
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.7.1/jquery.min.js"></script>
24
+
25
+ <link rel="stylesheet" href="style_videos.css">
26
+
27
+ <script type="text/javascript">
28
+ $(document).ready(function(){
29
+ $("#content").load("video_gen.html #moviegen_all");
30
+ $("#load_moveigen").click(function(){
31
+ $("#content").load("video_gen.html #moviegen_all");
32
+ });
33
+ $("#load_hunyuan_sora").click(function(){
34
+ $("#content").load("video_gen.html #hunyuan_sora_all");
35
+ });
36
+ $("#load_mochi_ltx").click(function(){
37
+ $("#content").load("video_gen.html #mochi_ltx_all");
38
+ });
39
+ $("#load_vgg1").click(function(){
40
+ $("#content").load("video_vgg.html #vgg1");
41
+ });
42
+ $("#load_vgg2").click(function(){
43
+ $("#content").load("video_vgg.html #vgg2");
44
+ });
45
+ $("#load_vgg3").click(function(){
46
+ $("#content").load("video_vgg.html #vgg3");
47
+ });
48
+ $("#load_vgg4").click(function(){
49
+ $("#content").load("video_vgg.html #vgg4");
50
+ });
51
+ $("#load_vgg5").click(function(){
52
+ $("#content").load("video_vgg.html #vgg5");
53
+ });
54
+ $("#load_vgg6").click(function(){
55
+ $("#content").load("video_vgg.html #vgg6");
56
+ });
57
+ $("#load_vgg_extra").click(function(){
58
+ $("#content").load("video_vgg.html #vgg_extra");
59
+ });
60
+ });
61
+ </script>
62
+ </head>
63
+ <body>
64
+ <h1 id="index" style="text-align: center;">Index</h1>
65
+ <p><b>(Click on the links to load the corresponding videos)</b> <span style="float:right;"><a href="index.html">Back to project page</a></span></p>
66
+
67
+ <ol>
68
+ <li>
69
+ <a href="#" id="load_moveigen">Comparisons with Movie Gen Audio on Videos Generated by MovieGen</a>
70
+ </li>
71
+ <li>
72
+ <a href="#" id="load_hunyuan_sora">Results on Videos Generated by Hunyuan and Sora</a>
73
+ </li>
74
+ <li>
75
+ <a href="#" id="load_mochi_ltx">Results on Videos Generated by Mochi 1 and LTX-Video</a>
76
+ </li>
77
+ <li>
78
+ On VGGSound
79
+ <ol>
80
+ <li><a id='load_vgg1' href="#">Example 1: Wolf howling</a></li>
81
+ <li><a id='load_vgg2' href="#">Example 2: Striking a golf ball</a></li>
82
+ <li><a id='load_vgg3' href="#">Example 3: Hitting a drum</a></li>
83
+ <li><a id='load_vgg4' href="#">Example 4: Dog barking</a></li>
84
+ <li><a id='load_vgg5' href="#">Example 5: Playing a string instrument</a></li>
85
+ <li><a id='load_vgg6' href="#">Example 6: A group of people playing tambourines</a></li>
86
+ <li><a id='load_vgg_extra' href="#">Extra results & failure cases</a></li>
87
+ </ol>
88
+ </li>
89
+ </ol>
90
+
91
+ <div id="content" class="container-fluid">
92
+
93
+ </div>
94
+ <br>
95
+ <br>
96
+
97
+ </body>
98
+ </html>
docs/video_vgg.html ADDED
@@ -0,0 +1,452 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <!-- Google tag (gtag.js) -->
5
+ <script async src="https://www.googletagmanager.com/gtag/js?id=G-0JKBJ3WRJZ"></script>
6
+ <script>
7
+ window.dataLayer = window.dataLayer || [];
8
+ function gtag(){dataLayer.push(arguments);}
9
+ gtag('js', new Date());
10
+ gtag('config', 'G-0JKBJ3WRJZ');
11
+ </script>
12
+
13
+ <link href='https://fonts.googleapis.com/css?family=Source+Sans+Pro' rel='stylesheet' type='text/css'>
14
+ <meta charset="UTF-8">
15
+ <title>MMAudio</title>
16
+
17
+ <meta name="viewport" content="width=device-width, initial-scale=1">
18
+ <!-- CSS only -->
19
+ <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.1/dist/css/bootstrap.min.css" rel="stylesheet"
20
+ integrity="sha384-+0n0xVW2eSR5OomGNYDnhzAbDsOXxcvSN1TPprVMTNDbiYZCxYbOOl7+AMvyTG2x" crossorigin="anonymous">
21
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
22
+
23
+ <link rel="stylesheet" href="style_videos.css">
24
+ </head>
25
+ <body>
26
+
27
+ <div id="vgg1">
28
+ <h2 style="text-align: center;">Comparisons with state-of-the-art methods in VGGSound</h2>
29
+ <p style="overflow: hidden;">
30
+ Example 1: Wolf howling.
31
+ <span style="float:right;"><a href="#index">Back to index</a></span>
32
+ </p>
33
+ <div class="row g-1">
34
+ <div class="col-sm-3">
35
+ <div class="video-header">Ground-truth</div>
36
+ <div class="video-container">
37
+ <iframe src="https://youtube.com/embed/9J_V74gqMUA"></iframe>
38
+ </div>
39
+ </div>
40
+ <div class="col-sm-3">
41
+ <div class="video-header">Ours</div>
42
+ <div class="video-container">
43
+ <iframe src="https://youtube.com/embed/P6O8IpjErPc"></iframe>
44
+ </div>
45
+ </div>
46
+ <div class="col-sm-3">
47
+ <div class="video-header">V2A-Mapper</div>
48
+ <div class="video-container">
49
+ <iframe src="https://youtube.com/embed/w-5eyqepvTk"></iframe>
50
+ </div>
51
+ </div>
52
+ <div class="col-sm-3">
53
+ <div class="video-header">FoleyCrafter</div>
54
+ <div class="video-container">
55
+ <iframe src="https://youtube.com/embed/VOLfoZlRkzo"></iframe>
56
+ </div>
57
+ </div>
58
+ </div>
59
+ <div class="row g-1">
60
+ <div class="col-sm-3">
61
+ <div class="video-header">Frieren</div>
62
+ <div class="video-container">
63
+ <iframe src="https://youtube.com/embed/49owKyA5Pa8"></iframe>
64
+ </div>
65
+ </div>
66
+ <div class="col-sm-3">
67
+ <div class="video-header">VATT</div>
68
+ <div class="video-container">
69
+ <iframe src="https://youtube.com/embed/QVtrFgbeGDM"></iframe>
70
+ </div>
71
+ </div>
72
+ <div class="col-sm-3">
73
+ <div class="video-header">V-AURA</div>
74
+ <div class="video-container">
75
+ <iframe src="https://youtube.com/embed/8r0uEfSNjvI"></iframe>
76
+ </div>
77
+ </div>
78
+ <div class="col-sm-3">
79
+ <div class="video-header">Seeing and Hearing</div>
80
+ <div class="video-container">
81
+ <iframe src="https://youtube.com/embed/bn-sLg2qulk"></iframe>
82
+ </div>
83
+ </div>
84
+ </div>
85
+ </div>
86
+
87
+ <div id="vgg2">
88
+ <h2 style="text-align: center;">Comparisons with state-of-the-art methods in VGGSound</h2>
89
+ <p style="overflow: hidden;">
90
+ Example 2: Striking a golf ball.
91
+ <span style="float:right;"><a href="#index">Back to index</a></span>
92
+ </p>
93
+
94
+ <div class="row g-1">
95
+ <div class="col-sm-3">
96
+ <div class="video-header">Ground-truth</div>
97
+ <div class="video-container">
98
+ <iframe src="https://youtube.com/embed/1hwSu42kkho"></iframe>
99
+ </div>
100
+ </div>
101
+ <div class="col-sm-3">
102
+ <div class="video-header">Ours</div>
103
+ <div class="video-container">
104
+ <iframe src="https://youtube.com/embed/kZibDoDCNxI"></iframe>
105
+ </div>
106
+ </div>
107
+ <div class="col-sm-3">
108
+ <div class="video-header">V2A-Mapper</div>
109
+ <div class="video-container">
110
+ <iframe src="https://youtube.com/embed/jgKfLBLhh7Y"></iframe>
111
+ </div>
112
+ </div>
113
+ <div class="col-sm-3">
114
+ <div class="video-header">FoleyCrafter</div>
115
+ <div class="video-container">
116
+ <iframe src="https://youtube.com/embed/Lfsx8mOPcJo"></iframe>
117
+ </div>
118
+ </div>
119
+ </div>
120
+ <div class="row g-1">
121
+ <div class="col-sm-3">
122
+ <div class="video-header">Frieren</div>
123
+ <div class="video-container">
124
+ <iframe src="https://youtube.com/embed/tz-LpbB0MBc"></iframe>
125
+ </div>
126
+ </div>
127
+ <div class="col-sm-3">
128
+ <div class="video-header">VATT</div>
129
+ <div class="video-container">
130
+ <iframe src="https://youtube.com/embed/RTDUHMi08n4"></iframe>
131
+ </div>
132
+ </div>
133
+ <div class="col-sm-3">
134
+ <div class="video-header">V-AURA</div>
135
+ <div class="video-container">
136
+ <iframe src="https://youtube.com/embed/N-3TDOsPnZQ"></iframe>
137
+ </div>
138
+ </div>
139
+ <div class="col-sm-3">
140
+ <div class="video-header">Seeing and Hearing</div>
141
+ <div class="video-container">
142
+ <iframe src="https://youtube.com/embed/QnsHnLn4gB0"></iframe>
143
+ </div>
144
+ </div>
145
+ </div>
146
+ </div>
147
+
148
+ <div id="vgg3">
149
+ <h2 style="text-align: center;">Comparisons with state-of-the-art methods in VGGSound</h2>
150
+ <p style="overflow: hidden;">
151
+ Example 3: Hitting a drum.
152
+ <span style="float:right;"><a href="#index">Back to index</a></span>
153
+ </p>
154
+
155
+ <div class="row g-1">
156
+ <div class="col-sm-3">
157
+ <div class="video-header">Ground-truth</div>
158
+ <div class="video-container">
159
+ <iframe src="https://youtube.com/embed/0oeIwq77w0Q"></iframe>
160
+ </div>
161
+ </div>
162
+ <div class="col-sm-3">
163
+ <div class="video-header">Ours</div>
164
+ <div class="video-container">
165
+ <iframe src="https://youtube.com/embed/-UtPV9ohuIM"></iframe>
166
+ </div>
167
+ </div>
168
+ <div class="col-sm-3">
169
+ <div class="video-header">V2A-Mapper</div>
170
+ <div class="video-container">
171
+ <iframe src="https://youtube.com/embed/9yivkgN-zwc"></iframe>
172
+ </div>
173
+ </div>
174
+ <div class="col-sm-3">
175
+ <div class="video-header">FoleyCrafter</div>
176
+ <div class="video-container">
177
+ <iframe src="https://youtube.com/embed/kkCsXPOlBvY"></iframe>
178
+ </div>
179
+ </div>
180
+ </div>
181
+ <div class="row g-1">
182
+ <div class="col-sm-3">
183
+ <div class="video-header">Frieren</div>
184
+ <div class="video-container">
185
+ <iframe src="https://youtube.com/embed/MbNKsVsuvig"></iframe>
186
+ </div>
187
+ </div>
188
+ <div class="col-sm-3">
189
+ <div class="video-header">VATT</div>
190
+ <div class="video-container">
191
+ <iframe src="https://youtube.com/embed/2yYviBjrpBw"></iframe>
192
+ </div>
193
+ </div>
194
+ <div class="col-sm-3">
195
+ <div class="video-header">V-AURA</div>
196
+ <div class="video-container">
197
+ <iframe src="https://youtube.com/embed/9yivkgN-zwc"></iframe>
198
+ </div>
199
+ </div>
200
+ <div class="col-sm-3">
201
+ <div class="video-header">Seeing and Hearing</div>
202
+ <div class="video-container">
203
+ <iframe src="https://youtube.com/embed/6dnyQt4Fuhs"></iframe>
204
+ </div>
205
+ </div>
206
+ </div>
207
+ </div>
208
+ </div>
209
+
210
+ <div id="vgg4">
211
+ <h2 style="text-align: center;">Comparisons with state-of-the-art methods in VGGSound</h2>
212
+ <p style="overflow: hidden;">
213
+ Example 4: Dog barking.
214
+ <span style="float:right;"><a href="#index">Back to index</a></span>
215
+ </p>
216
+
217
+ <div class="row g-1">
218
+ <div class="col-sm-3">
219
+ <div class="video-header">Ground-truth</div>
220
+ <div class="video-container">
221
+ <iframe src="https://youtube.com/embed/ckaqvTyMYAw"></iframe>
222
+ </div>
223
+ </div>
224
+ <div class="col-sm-3">
225
+ <div class="video-header">Ours</div>
226
+ <div class="video-container">
227
+ <iframe src="https://youtube.com/embed/_aRndFZzZ-I"></iframe>
228
+ </div>
229
+ </div>
230
+ <div class="col-sm-3">
231
+ <div class="video-header">V2A-Mapper</div>
232
+ <div class="video-container">
233
+ <iframe src="https://youtube.com/embed/mNCISP3LBl0"></iframe>
234
+ </div>
235
+ </div>
236
+ <div class="col-sm-3">
237
+ <div class="video-header">FoleyCrafter</div>
238
+ <div class="video-container">
239
+ <iframe src="https://youtube.com/embed/phZBQ3L7foE"></iframe>
240
+ </div>
241
+ </div>
242
+ </div>
243
+ <div class="row g-1">
244
+ <div class="col-sm-3">
245
+ <div class="video-header">Frieren</div>
246
+ <div class="video-container">
247
+ <iframe src="https://youtube.com/embed/Sb5Mg1-ORao"></iframe>
248
+ </div>
249
+ </div>
250
+ <div class="col-sm-3">
251
+ <div class="video-header">VATT</div>
252
+ <div class="video-container">
253
+ <iframe src="https://youtube.com/embed/eHmAGOmtDDg"></iframe>
254
+ </div>
255
+ </div>
256
+ <div class="col-sm-3">
257
+ <div class="video-header">V-AURA</div>
258
+ <div class="video-container">
259
+ <iframe src="https://youtube.com/embed/NEGa3krBrm0"></iframe>
260
+ </div>
261
+ </div>
262
+ <div class="col-sm-3">
263
+ <div class="video-header">Seeing and Hearing</div>
264
+ <div class="video-container">
265
+ <iframe src="https://youtube.com/embed/aO0EAXlwE7A"></iframe>
266
+ </div>
267
+ </div>
268
+ </div>
269
+ </div>
270
+
271
+ <div id="vgg5">
272
+ <h2 style="text-align: center;">Comparisons with state-of-the-art methods in VGGSound</h2>
273
+ <p style="overflow: hidden;">
274
+ Example 5: Playing a string instrument.
275
+ <span style="float:right;"><a href="#index">Back to index</a></span>
276
+ </p>
277
+
278
+ <div class="row g-1">
279
+ <div class="col-sm-3">
280
+ <div class="video-header">Ground-truth</div>
281
+ <div class="video-container">
282
+ <iframe src="https://youtube.com/embed/KP1QhWauIOc"></iframe>
283
+ </div>
284
+ </div>
285
+ <div class="col-sm-3">
286
+ <div class="video-header">Ours</div>
287
+ <div class="video-container">
288
+ <iframe src="https://youtube.com/embed/ovaJhWSquYE"></iframe>
289
+ </div>
290
+ </div>
291
+ <div class="col-sm-3">
292
+ <div class="video-header">V2A-Mapper</div>
293
+ <div class="video-container">
294
+ <iframe src="https://youtube.com/embed/N723FS9lcy8"></iframe>
295
+ </div>
296
+ </div>
297
+ <div class="col-sm-3">
298
+ <div class="video-header">FoleyCrafter</div>
299
+ <div class="video-container">
300
+ <iframe src="https://youtube.com/embed/t0N4ZAAXo58"></iframe>
301
+ </div>
302
+ </div>
303
+ </div>
304
+ <div class="row g-1">
305
+ <div class="col-sm-3">
306
+ <div class="video-header">Frieren</div>
307
+ <div class="video-container">
308
+ <iframe src="https://youtube.com/embed/8YSRs03QNNA"></iframe>
309
+ </div>
310
+ </div>
311
+ <div class="col-sm-3">
312
+ <div class="video-header">VATT</div>
313
+ <div class="video-container">
314
+ <iframe src="https://youtube.com/embed/vOpMz55J1kY"></iframe>
315
+ </div>
316
+ </div>
317
+ <div class="col-sm-3">
318
+ <div class="video-header">V-AURA</div>
319
+ <div class="video-container">
320
+ <iframe src="https://youtube.com/embed/9JHC75vr9h0"></iframe>
321
+ </div>
322
+ </div>
323
+ <div class="col-sm-3">
324
+ <div class="video-header">Seeing and Hearing</div>
325
+ <div class="video-container">
326
+ <iframe src="https://youtube.com/embed/9w0JckNzXmY"></iframe>
327
+ </div>
328
+ </div>
329
+ </div>
330
+ </div>
331
+
332
+ <div id="vgg6">
333
+ <h2 style="text-align: center;">Comparisons with state-of-the-art methods in VGGSound</h2>
334
+ <p style="overflow: hidden;">
335
+ Example 6: A group of people playing tambourines.
336
+ <span style="float:right;"><a href="#index">Back to index</a></span>
337
+ </p>
338
+
339
+ <div class="row g-1">
340
+ <div class="col-sm-3">
341
+ <div class="video-header">Ground-truth</div>
342
+ <div class="video-container">
343
+ <iframe src="https://youtube.com/embed/mx6JLxzUkRc"></iframe>
344
+ </div>
345
+ </div>
346
+ <div class="col-sm-3">
347
+ <div class="video-header">Ours</div>
348
+ <div class="video-container">
349
+ <iframe src="https://youtube.com/embed/oLirHhP9Su8"></iframe>
350
+ </div>
351
+ </div>
352
+ <div class="col-sm-3">
353
+ <div class="video-header">V2A-Mapper</div>
354
+ <div class="video-container">
355
+ <iframe src="https://youtube.com/embed/HkLkHMqptv0"></iframe>
356
+ </div>
357
+ </div>
358
+ <div class="col-sm-3">
359
+ <div class="video-header">FoleyCrafter</div>
360
+ <div class="video-container">
361
+ <iframe src="https://youtube.com/embed/rpHiiODjmNU"></iframe>
362
+ </div>
363
+ </div>
364
+ </div>
365
+ <div class="row g-1">
366
+ <div class="col-sm-3">
367
+ <div class="video-header">Frieren</div>
368
+ <div class="video-container">
369
+ <iframe src="https://youtube.com/embed/1mVD3fJ0LpM"></iframe>
370
+ </div>
371
+ </div>
372
+ <div class="col-sm-3">
373
+ <div class="video-header">VATT</div>
374
+ <div class="video-container">
375
+ <iframe src="https://youtube.com/embed/yjVFnJiEJlw"></iframe>
376
+ </div>
377
+ </div>
378
+ <div class="col-sm-3">
379
+ <div class="video-header">V-AURA</div>
380
+ <div class="video-container">
381
+ <iframe src="https://youtube.com/embed/neVeMSWtRkU"></iframe>
382
+ </div>
383
+ </div>
384
+ <div class="col-sm-3">
385
+ <div class="video-header">Seeing and Hearing</div>
386
+ <div class="video-container">
387
+ <iframe src="https://youtube.com/embed/EUE7YwyVWz8"></iframe>
388
+ </div>
389
+ </div>
390
+ </div>
391
+ </div>
392
+
393
+ <div id="vgg_extra">
394
+ <h2 style="text-align: center;">Comparisons with state-of-the-art methods in VGGSound</h2>
395
+ <p style="overflow: hidden;">
396
+ <span style="float:right;"><a href="#index">Back to index</a></span>
397
+ </p>
398
+
399
+ <div class="row g-1">
400
+ <div class="col-sm-3">
401
+ <div class="video-header">Moving train</div>
402
+ <div class="video-container">
403
+ <iframe src="https://youtube.com/embed/Ta6H45rBzJc"></iframe>
404
+ </div>
405
+ </div>
406
+ <div class="col-sm-3">
407
+ <div class="video-header">Water splashing</div>
408
+ <div class="video-container">
409
+ <iframe src="https://youtube.com/embed/hl6AtgHXpb4"></iframe>
410
+ </div>
411
+ </div>
412
+ <div class="col-sm-3">
413
+ <div class="video-header">Skateboarding</div>
414
+ <div class="video-container">
415
+ <iframe src="https://youtube.com/embed/n4sCNi_9buI"></iframe>
416
+ </div>
417
+ </div>
418
+ <div class="col-sm-3">
419
+ <div class="video-header">Synchronized clapping</div>
420
+ <div class="video-container">
421
+ <iframe src="https://youtube.com/embed/oxexfpLn7FE"></iframe>
422
+ </div>
423
+ </div>
424
+ </div>
425
+
426
+ <br><br>
427
+
428
+ <div id="extra-failure">
429
+ <h2 style="text-align: center;">Failure cases</h2>
430
+ <p style="overflow: hidden;">
431
+ <span style="float:right;"><a href="#index">Back to index</a></span>
432
+ </p>
433
+
434
+ <div class="row g-1">
435
+ <div class="col-sm-6">
436
+ <div class="video-header">Human speech</div>
437
+ <div class="video-container">
438
+ <iframe src="https://youtube.com/embed/nx0CyrDu70Y"></iframe>
439
+ </div>
440
+ </div>
441
+ <div class="col-sm-6">
442
+ <div class="video-header">Unfamiliar vision input</div>
443
+ <div class="video-container">
444
+ <iframe src="https://youtube.com/embed/hfnAqmK3X7w"></iframe>
445
+ </div>
446
+ </div>
447
+ </div>
448
+ </div>
449
+ </div>
450
+
451
+ </body>
452
+ </html>