English
Wubyte commited on
Commit
1aed655
·
verified ·
1 Parent(s): ce6255a

Upload ckpt & readme

Browse files
.gitattributes CHANGED
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ assets/96F32G.png filter=lfs diff=lfs merge=lfs -text
37
+ assets/allF-1G.png filter=lfs diff=lfs merge=lfs -text
38
+ assets/architecture.png filter=lfs diff=lfs merge=lfs -text
39
+ assets/performance.png filter=lfs diff=lfs merge=lfs -text
LICENSE ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Copyright 2025 ByteDance Ltd. and/or its affiliates.
2
+ // All rights reserved.
3
+ // Licensed under the BSD 3-Clause Clear License (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ // https://choosealicense.com/licenses/bsd-3-clause-clear/
7
+ // Unless required by applicable law or agreed to in writing, software
8
+ // distributed under the License is distributed on an "AS IS" BASIS,
9
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10
+ // See the License for the specific language governing permissions and
11
+ // limitations under the License.
12
+
13
+ // Redistribution and use in source and binary forms, with or without
14
+ // modification, are permitted (subject to the limitations in the disclaimer
15
+ // below) provided that the following conditions are met:
16
+
17
+ // * Redistributions of source code must retain the above copyright notice,
18
+ // this list of conditions and the following disclaimer.
19
+ // * Redistributions in binary form must reproduce the above copyright notice,
20
+ // this list of conditions and the following disclaimer in the documentation
21
+ // and/or other materials provided with the distribution.
22
+ // * Neither the name of ByteDance Ltd. and/or its affiliates Communications, Inc nor the names of its
23
+ // contributors may be used to endorse or promote products derived from this
24
+ // software without specific prior written permission.
25
+
26
+ // NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
27
+ // THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
28
+ // CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
29
+ // NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
30
+ // PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
31
+ // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32
+ // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33
+ // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
34
+ // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
35
+ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
36
+ // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
37
+ // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
README.md CHANGED
@@ -1,3 +1,132 @@
1
- ---
2
- license: bsd-3-clause-clear
3
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!-- Copyright 2025 ByteDance Ltd. and/or its affiliates.
2
+ All rights reserved.
3
+ Licensed under the BSD 3-Clause Clear License (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+ https://choosealicense.com/licenses/bsd-3-clause-clear/
7
+ Unless required by applicable law or agreed to in writing, software
8
+ distributed under the License is distributed on an "AS IS" BASIS,
9
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10
+ See the License for the specific language governing permissions and
11
+ limitations under the License.
12
+
13
+ Redistribution and use in source and binary forms, with or without
14
+ modification, are permitted (subject to the limitations in the disclaimer
15
+ below) provided that the following conditions are met:
16
+
17
+ * Redistributions of source code must retain the above copyright notice,
18
+ this list of conditions and the following disclaimer.
19
+ * Redistributions in binary form must reproduce the above copyright notice,
20
+ this list of conditions and the following disclaimer in the documentation
21
+ and/or other materials provided with the distribution.
22
+ * Neither the name of ByteDance Ltd. and/or its affiliates nor the names of its
23
+ contributors may be used to endorse or promote products derived from this
24
+ software without specific prior written permission.
25
+
26
+ NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
27
+ THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
28
+ CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
29
+ NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
30
+ PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
31
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
34
+ OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
35
+ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
36
+ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
37
+ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -->
38
+
39
+ <div align="center">
40
+
41
+ # **NEVC-1.0** <br>(EHVC: Efficient Hierarchical Reference and Quality Structure for Neural Video Coding)
42
+
43
+ <div align="center">
44
+ <img src="./assets/performance.png" alt="Performance comparison" width="60%" style="max-width: 100%;" height="auto">
45
+ </div>
46
+
47
+ </div>
48
+
49
+ <div align="left">
50
+
51
+ ## 📝 Introduction
52
+ This repository provides the pretrained model weights for **NEVC-1.0**, which integrates contributions from **EHVC (Efficient Hierarchical Reference and Quality Structure for Neural Video Coding)** — one of the core components of the framework.
53
+ **EHVC** introduces a hierarchical reference and quality structure that significantly improves both compression efficiency and rate–distortion performance.
54
+ The corresponding code repository can be found here: [NEVC-1.0-EHVC](https://github.com/bytedance/NEVC).
55
+
56
+ Key designs of **EHVC** include:
57
+ - **Hierarchical multi-reference:** Resolves reference–quality mismatches using a hierarchical reference structure and a multi-reference scheme, optimized for low-delay configurations.
58
+ - **Lookahead mechanism:** Enhances encoder-side context by leveraging forward features, thereby improving prediction accuracy and compression.
59
+ - **Layer-wise quantization scale with random quality training:** Provides a flexible and efficient quality structure that adapts during training, resulting in improved encoding performance.
60
+
61
+ ---
62
+
63
+ ## 🔧 Models
64
+ EHVC uses two models: the intra model and the inter model.
65
+ - The **intra model** handles intra-frame coding.
66
+ - The **inter model** is responsible for inter-frame (predictive) coding.
67
+
68
+ ### Intra Model
69
+ The main contributions of NEVC-1.0 focus on inter coding.
70
+ For intra coding, we directly adopt the pretrained model **`cvpr2023_image_psnr.pth.tar`** from [DCVC-DC](https://github.com/microsoft/DCVC/blob/main/DCVC-family/DCVC-DC/checkpoints/download.py), without further training.
71
+
72
+ ### Inter Model
73
+ The inter model of NEVC-1.0 is provided at **`/models/nevc1.0_inter.pth.tar`**.
74
+ The architecture of the inter model is illustrated below:
75
+
76
+ <div align="center">
77
+ <img src="./assets/architecture.png" alt="Inter model architecture" width="50%" style="max-width: 100%;" height="auto">
78
+ </div>
79
+
80
+ ---
81
+
82
+ ## 📊 Experimental Results
83
+ ### Objective Comparison
84
+ <div align="center">
85
+
86
+ **BD-Rate (%) comparison for PSNR**
87
+ Anchor: VTM-23.4 LDB.
88
+ All codecs tested with 96 frames and intra-period = 32.
89
+
90
+ <img src="./assets/96F32G.png" alt="BD-Rate 96F32G" width="50%" style="max-width: 100%;" height="auto">
91
+
92
+ **Rate–Distortion curves** on HEVC B, HEVC C, UVG, and MCL-JCV datasets.
93
+ Tested with 96 frames and intra-period = 32.
94
+
95
+ <img src="./assets/96F32G_curve.png" alt="RD curves 96F32G" width="80%" style="max-width: 100%;" height="auto">
96
+
97
+ **BD-Rate (%) comparison for PSNR**
98
+ Anchor: VTM-23.4 LDB.
99
+ All codecs tested with full sequences and intra-period = -1.
100
+
101
+ <img src="./assets/allF-1G.png" alt="BD-Rate allF-1G" width="50%" style="max-width: 100%;" height="auto">
102
+
103
+ **Rate–Distortion curves** on HEVC B, HEVC C, UVG, and MCL-JCV datasets.
104
+ Tested with full sequences and intra-period = -1.
105
+
106
+ <img src="./assets/allF-1G_curve.png" alt="RD curves allF-1G" width="80%" style="max-width: 100%;" height="auto">
107
+
108
+ </div>
109
+
110
+ ---
111
+
112
+ ## 📜 Citation
113
+ If you find **NEVC-1.0** useful in your research or projects, please cite the following paper:
114
+
115
+ - **EHVC: Efficient Hierarchical Reference and Quality Structure for Neural Video Coding**
116
+ Junqi Liao, Yaojun Wu, Chaoyi Lin, Zhipin Deng, Li Li, Dong Liu, Xiaoyan Sun.
117
+ *Proceedings of the 33rd ACM International Conference on Multimedia (ACM MM 2025).*
118
+
119
+ ```bibtex
120
+ @inproceedings{liao2025ehvc,
121
+ title={EHVC: Efficient Hierarchical Reference and Quality Structure for Neural Video Coding},
122
+ author={Liao, Junqi and Wu, Yaojun and Lin, Chaoyi and Deng, Zhipin and Li, Li and Liu, Dong and Sun, Xiaoyan},
123
+ booktitle={Proceedings of the 33rd ACM International Conference on Multimedia},
124
+ year={2025}
125
+ }
126
+ ```
127
+
128
+ ---
129
+
130
+
131
+ ## 🙌 Acknowledgement
132
+ The intra model of this project is based on [DCVC-DC](https://github.com/microsoft/DCVC/blob/main/DCVC-family/DCVC-DC/checkpoints/download.py).
assets/96F32G.png ADDED

Git LFS Details

  • SHA256: 4360324a3c8765209146c2de01fec8d8bf9a36ece408fd28745432eccf3d536e
  • Pointer size: 131 Bytes
  • Size of remote file: 242 kB
assets/96F32G_curve.png ADDED
assets/allF-1G.png ADDED

Git LFS Details

  • SHA256: f7f4252f8950867f6a56deb4fa784e8068c8626a1e82a4a9296d4b9f43b41782
  • Pointer size: 131 Bytes
  • Size of remote file: 245 kB
assets/allF-1G_curve.png ADDED
assets/architecture.png ADDED

Git LFS Details

  • SHA256: 7b04c9cbf4b46695b72f4a059d742e1f803bcb0a5b5b7d34a6de1b28764246c8
  • Pointer size: 131 Bytes
  • Size of remote file: 124 kB
assets/performance.png ADDED

Git LFS Details

  • SHA256: 9efeb73275f431d550d680593d4d80b3d623d4ea022fe939fc109f98ca7c6107
  • Pointer size: 131 Bytes
  • Size of remote file: 151 kB
licenses/LICENSE-DCVC.txt ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Copyright (c) Microsoft Corporation.
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
models/nevc1.0_inter.pth.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:991679f77367bda9d0d65045155607b9b98170a8e23c71ceaba3ab5dfdeb7cb9
3
+ size 81926714
models/nevc1.0_intra.pth.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:75907ad25a572e64dc58fd832c0c8877c0a39734d77b80383066f656240eec37
3
+ size 124096679